1 /*
2 * Copyright (c) 1998-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
29 /*
30 * Copyright (c) 1982, 1986, 1988, 1991, 1993
31 * The Regents of the University of California. All rights reserved.
32 *
33 * Redistribution and use in source and binary forms, with or without
34 * modification, are permitted provided that the following conditions
35 * are met:
36 * 1. Redistributions of source code must retain the above copyright
37 * notice, this list of conditions and the following disclaimer.
38 * 2. Redistributions in binary form must reproduce the above copyright
39 * notice, this list of conditions and the following disclaimer in the
40 * documentation and/or other materials provided with the distribution.
41 * 3. All advertising materials mentioning features or use of this software
42 * must display the following acknowledgement:
43 * This product includes software developed by the University of
44 * California, Berkeley and its contributors.
45 * 4. Neither the name of the University nor the names of its contributors
46 * may be used to endorse or promote products derived from this software
47 * without specific prior written permission.
48 *
49 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
50 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
51 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
52 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
53 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
54 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
55 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
56 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
57 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
58 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
59 * SUCH DAMAGE.
60 *
61 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94
62 */
63 /*
64 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
65 * support for mandatory and extensible security protections. This notice
66 * is included in support of clause 2.2 (b) of the Apple Public License,
67 * Version 2.0.
68 */
69
70 #include <ptrauth.h>
71
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/malloc.h>
75 #include <sys/mbuf.h>
76 #include <sys/kernel.h>
77 #include <sys/sysctl.h>
78 #include <sys/syslog.h>
79 #include <sys/protosw.h>
80 #include <sys/domain.h>
81 #include <sys/queue.h>
82 #include <sys/proc.h>
83 #include <sys/filedesc.h>
84 #include <sys/file_internal.h>
85
86 #include <vm/vm_kern_xnu.h>
87
88 #include <dev/random/randomdev.h>
89
90 #include <kern/kern_types.h>
91 #include <kern/simple_lock.h>
92 #include <kern/queue.h>
93 #include <kern/sched_prim.h>
94 #include <kern/backtrace.h>
95 #include <kern/percpu.h>
96 #include <kern/zalloc.h>
97
98 #include <libkern/OSDebug.h>
99 #include <libkern/libkern.h>
100
101 #include <os/log.h>
102 #include <os/ptrtools.h>
103
104 #include <IOKit/IOMapper.h>
105
106 #include <machine/limits.h>
107 #include <machine/machine_routines.h>
108
109 #if CONFIG_MBUF_MCACHE
110 #include <sys/mcache.h>
111 #endif /* CONFIG_MBUF_MCACHE */
112 #include <net/ntstat.h>
113
114 #include <net/droptap.h>
115
116 #if INET
117 extern int dump_tcp_reass_qlen(char *, int);
118 extern int tcp_reass_qlen_space(struct socket *);
119 #endif /* INET */
120
121 #if MPTCP
122 extern int dump_mptcp_reass_qlen(char *, int);
123 #endif /* MPTCP */
124
125
126 #if NETWORKING
127 extern int dlil_dump_top_if_qlen(char *, int);
128 #endif /* NETWORKING */
129
130 #if CONFIG_MBUF_MCACHE
131 /*
132 * MBUF IMPLEMENTATION NOTES.
133 *
134 * There is a total of 5 per-CPU caches:
135 *
136 * MC_MBUF:
137 * This is a cache of rudimentary objects of _MSIZE in size; each
138 * object represents an mbuf structure. This cache preserves only
139 * the m_type field of the mbuf during its transactions.
140 *
141 * MC_CL:
142 * This is a cache of rudimentary objects of MCLBYTES in size; each
143 * object represents a mcluster structure. This cache does not
144 * preserve the contents of the objects during its transactions.
145 *
146 * MC_BIGCL:
147 * This is a cache of rudimentary objects of MBIGCLBYTES in size; each
148 * object represents a mbigcluster structure. This cache does not
149 * preserve the contents of the objects during its transaction.
150 *
151 * MC_MBUF_CL:
152 * This is a cache of mbufs each having a cluster attached to it.
153 * It is backed by MC_MBUF and MC_CL rudimentary caches. Several
154 * fields of the mbuf related to the external cluster are preserved
155 * during transactions.
156 *
157 * MC_MBUF_BIGCL:
158 * This is a cache of mbufs each having a big cluster attached to it.
159 * It is backed by MC_MBUF and MC_BIGCL rudimentary caches. Several
160 * fields of the mbuf related to the external cluster are preserved
161 * during transactions.
162 *
163 * OBJECT ALLOCATION:
164 *
165 * Allocation requests are handled first at the per-CPU (mcache) layer
166 * before falling back to the slab layer. Performance is optimal when
167 * the request is satisfied at the CPU layer because global data/lock
168 * never gets accessed. When the slab layer is entered for allocation,
169 * the slab freelist will be checked first for available objects before
170 * the VM backing store is invoked. Slab layer operations are serialized
171 * for all of the caches as the mbuf global lock is held most of the time.
172 * Allocation paths are different depending on the class of objects:
173 *
174 * a. Rudimentary object:
175 *
176 * { m_get_common(), m_clattach(), m_mclget(),
177 * m_mclalloc(), m_bigalloc(), m_copym_with_hdrs(),
178 * composite object allocation }
179 * | ^
180 * | |
181 * | +-----------------------+
182 * v |
183 * mcache_alloc/mcache_alloc_ext() mbuf_slab_audit()
184 * | ^
185 * v |
186 * [CPU cache] -------> (found?) -------+
187 * | |
188 * v |
189 * mbuf_slab_alloc() |
190 * | |
191 * v |
192 * +---------> [freelist] -------> (found?) -------+
193 * | |
194 * | v
195 * | m_clalloc()
196 * | |
197 * | v
198 * +---<<---- kmem_mb_alloc()
199 *
200 * b. Composite object:
201 *
202 * { m_getpackets_internal(), m_allocpacket_internal() }
203 * | ^
204 * | |
205 * | +------ (done) ---------+
206 * v |
207 * mcache_alloc/mcache_alloc_ext() mbuf_cslab_audit()
208 * | ^
209 * v |
210 * [CPU cache] -------> (found?) -------+
211 * | |
212 * v |
213 * mbuf_cslab_alloc() |
214 * | |
215 * v |
216 * [freelist] -------> (found?) -------+
217 * | |
218 * v |
219 * (rudimentary object) |
220 * mcache_alloc/mcache_alloc_ext() ------>>-----+
221 *
222 * Auditing notes: If auditing is enabled, buffers will be subjected to
223 * integrity checks by the audit routine. This is done by verifying their
224 * contents against DEADBEEF (free) pattern before returning them to caller.
225 * As part of this step, the routine will also record the transaction and
226 * pattern-fill the buffers with BADDCAFE (uninitialized) pattern. It will
227 * also restore any constructed data structure fields if necessary.
228 *
229 * OBJECT DEALLOCATION:
230 *
231 * Freeing an object simply involves placing it into the CPU cache; this
232 * pollutes the cache to benefit subsequent allocations. The slab layer
233 * will only be entered if the object is to be purged out of the cache.
234 * During normal operations, this happens only when the CPU layer resizes
235 * its bucket while it's adjusting to the allocation load. Deallocation
236 * paths are different depending on the class of objects:
237 *
238 * a. Rudimentary object:
239 *
240 * { m_free(), m_freem_list(), composite object deallocation }
241 * | ^
242 * | |
243 * | +------ (done) ---------+
244 * v |
245 * mcache_free/mcache_free_ext() |
246 * | |
247 * v |
248 * mbuf_slab_audit() |
249 * | |
250 * v |
251 * [CPU cache] ---> (not purging?) -----+
252 * | |
253 * v |
254 * mbuf_slab_free() |
255 * | |
256 * v |
257 * [freelist] ----------->>------------+
258 * (objects get purged to VM only on demand)
259 *
260 * b. Composite object:
261 *
262 * { m_free(), m_freem_list() }
263 * | ^
264 * | |
265 * | +------ (done) ---------+
266 * v |
267 * mcache_free/mcache_free_ext() |
268 * | |
269 * v |
270 * mbuf_cslab_audit() |
271 * | |
272 * v |
273 * [CPU cache] ---> (not purging?) -----+
274 * | |
275 * v |
276 * mbuf_cslab_free() |
277 * | |
278 * v |
279 * [freelist] ---> (not purging?) -----+
280 * | |
281 * v |
282 * (rudimentary object) |
283 * mcache_free/mcache_free_ext() ------->>------+
284 *
285 * Auditing notes: If auditing is enabled, the audit routine will save
286 * any constructed data structure fields (if necessary) before filling the
287 * contents of the buffers with DEADBEEF (free) pattern and recording the
288 * transaction. Buffers that are freed (whether at CPU or slab layer) are
289 * expected to contain the free pattern.
290 *
291 * DEBUGGING:
292 *
293 * Debugging can be enabled by adding "mbuf_debug=0x3" to boot-args; this
294 * translates to the mcache flags (MCF_VERIFY | MCF_AUDIT). Additionally,
295 * the CPU layer cache can be disabled by setting the MCF_NOCPUCACHE flag,
296 * i.e. modify the boot argument parameter to "mbuf_debug=0x13". Leak
297 * detection may also be disabled by setting the MCF_NOLEAKLOG flag, e.g.
298 * "mbuf_debug=0x113". Note that debugging consumes more CPU and memory.
299 *
300 * Each object is associated with exactly one mcache_audit_t structure that
301 * contains the information related to its last buffer transaction. Given
302 * an address of an object, the audit structure can be retrieved by finding
303 * the position of the object relevant to the base address of the cluster:
304 *
305 * +------------+ +=============+
306 * | mbuf addr | | mclaudit[i] |
307 * +------------+ +=============+
308 * | | cl_audit[0] |
309 * i = MTOBG(addr) +-------------+
310 * | +-----> | cl_audit[1] | -----> mcache_audit_t
311 * b = BGTOM(i) | +-------------+
312 * | | | ... |
313 * x = MCLIDX(b, addr) | +-------------+
314 * | | | cl_audit[7] |
315 * +-----------------+ +-------------+
316 * (e.g. x == 1)
317 *
318 * The mclaudit[] array is allocated at initialization time, but its contents
319 * get populated when the corresponding cluster is created. Because a page
320 * can be turned into NMBPG number of mbufs, we preserve enough space for the
321 * mbufs so that there is a 1-to-1 mapping between them. A page that never
322 * gets (or has not yet) turned into mbufs will use only cl_audit[0] with the
323 * remaining entries unused. For 16KB cluster, only one entry from the first
324 * page is allocated and used for the entire object.
325 */
326 #else
327 /*
328 * MBUF IMPLEMENTATION NOTES (using zalloc).
329 *
330 * There are a total of 4 zones and 3 zcaches.
331 *
332 * MC_MBUF:
333 * This is a zone of rudimentary objects of _MSIZE in size; each
334 * object represents an mbuf structure. This cache preserves only
335 * the m_type field of the mbuf during its transactions.
336 *
337 * MC_CL:
338 * This is a zone of rudimentary objects of MCLBYTES in size; each
339 * object represents a mcluster structure. This cache does not
340 * preserve the contents of the objects during its transactions.
341 *
342 * MC_BIGCL:
343 * This is a zone of rudimentary objects of MBIGCLBYTES in size; each
344 * object represents a mbigcluster structure. This cache does not
345 * preserve the contents of the objects during its transaction.
346 *
347 * MC_16KCL:
348 * This is a zone of rudimentary objects of M16KCLBYTES in size; each
349 * object represents a m16kcluster structure. This cache does not
350 * preserve the contents of the objects during its transaction.
351 *
352 * MC_MBUF_CL:
353 * This is a cache of mbufs each having a cluster attached to it.
354 * It is backed by MC_MBUF and MC_CL rudimentary caches. Several
355 * fields of the mbuf related to the external cluster are preserved
356 * during transactions.
357 *
358 * MC_MBUF_BIGCL:
359 * This is a cache of mbufs each having a big cluster attached to it.
360 * It is backed by MC_MBUF and MC_BIGCL rudimentary caches. Several
361 * fields of the mbuf related to the external cluster are preserved
362 * during transactions.
363 *
364 * MC_MBUF_16KCL:
365 * This is a cache of mbufs each having a big cluster attached to it.
366 * It is backed by MC_MBUF and MC_16KCL rudimentary caches. Several
367 * fields of the mbuf related to the external cluster are preserved
368 * during transactions.
369 *
370 * OBJECT ALLOCATION:
371 *
372 * Allocation requests are handled first at the zalloc per-CPU layer
373 * before falling back to the zalloc depot. Performance is optimal when
374 * the request is satisfied at the CPU layer. zalloc has an additional
375 * overflow layer called the depot, not pictured in the diagram below.
376 *
377 * Allocation paths are different depending on the class of objects:
378 *
379 * a. Rudimentary object:
380 *
381 * { m_get_common(), m_clattach(), m_mclget(),
382 * m_mclalloc(), m_bigalloc(), m_copym_with_hdrs(),
383 * composite object allocation }
384 * | ^
385 * | |
386 * | +------- (done) --------+
387 * v |
388 * zalloc_flags/zalloc_n() KASAN
389 * | ^
390 * v |
391 * +----> [zalloc per-CPU cache] -----> (found?) --+
392 * | | |
393 * | v |
394 * | [zalloc recirculation layer] --> (found?) ---+
395 * | |
396 * | v
397 * +--<<-- [zone backing store]
398 *
399 * b. Composite object:
400 *
401 * { m_getpackets_internal(), m_allocpacket_internal() }
402 * | ^
403 * | |
404 * | +------ (done) ---------+
405 * v |
406 * mz_composite_alloc() KASAN
407 * | ^
408 * v |
409 * zcache_alloc_n() |
410 * | |
411 * v |
412 * [zalloc per-CPU cache] --> mark_valid() ---+
413 * | |
414 * v |
415 * [zalloc recirculation layer] -> mark_valid() -+
416 * | |
417 * v |
418 * mz_composite_build() |
419 * | |
420 * v |
421 * (rudimentary objects) |
422 * zalloc_id() ---------------->>-----+
423 *
424 * Auditing notes: If KASAN enabled, buffers will be subjected to
425 * integrity checks by the AddressSanitizer.
426 *
427 * OBJECT DEALLOCATION:
428 *
429 * Freeing an object simply involves placing it into the CPU cache; this
430 * pollutes the cache to benefit subsequent allocations. The depot
431 * will only be entered if the object is to be purged out of the cache.
432 * Objects may be purged based on the overall memory pressure or
433 * during zone garbage collection.
434 * To improve performance, objects are not zero-filled when freed
435 * as it's custom for other zalloc zones.
436 *
437 * Deallocation paths are different depending on the class of objects:
438 *
439 * a. Rudimentary object:
440 *
441 * { m_free(), m_freem_list(), composite object deallocation }
442 * | ^
443 * | |
444 * | +------ (done) ---------+
445 * v |
446 * zfree_nozero() |
447 * | |
448 * v |
449 * KASAN |
450 * | |
451 * v |
452 * [zalloc per-CPU cache] -> (not purging?) --+
453 * | |
454 * v |
455 * [zalloc recirculation layer] --->>----------+
456 *
457 *
458 * b. Composite object:
459 *
460 * { m_free(), m_freem_list() }
461 * | ^
462 * | |
463 * | +------ (done) ---------+
464 * v |
465 * mz_composite_free() |
466 * | |
467 * v |
468 * zcache_free_n() |
469 * | |
470 * v |
471 * KASAN |
472 * | |
473 * v |
474 * [zalloc per-CPU cache] -> mark_invalid() --+
475 * | |
476 * v |
477 * mz_composite_destroy() |
478 * | |
479 * v |
480 * (rudimentary object) |
481 * zfree_nozero() -------------->>------+
482 *
483 * Auditing notes: If KASAN enabled, buffers will be subjected to
484 * integrity checks by the AddressSanitizer.
485 *
486 * DEBUGGING:
487 *
488 * Debugging mbufs can be done by booting a KASAN enabled kernel.
489 */
490
491 #endif /* CONFIG_MBUF_MCACHE */
492
493 /* TODO: should be in header file */
494 /* kernel translater */
495 extern ppnum_t pmap_find_phys(pmap_t pmap, addr64_t va);
496 extern vm_map_t mb_map; /* special map */
497
498 #if CONFIG_MBUF_MCACHE
499 static uint32_t mb_kmem_contig_failed;
500 static uint32_t mb_kmem_failed;
501 static uint32_t mb_kmem_one_failed;
502 /* Timestamp of allocation failures. */
503 static uint64_t mb_kmem_contig_failed_ts;
504 static uint64_t mb_kmem_failed_ts;
505 static uint64_t mb_kmem_one_failed_ts;
506 static uint64_t mb_kmem_contig_failed_size;
507 static uint64_t mb_kmem_failed_size;
508 static uint32_t mb_kmem_stats[6];
509 #endif /* CONFIG_MBUF_MCACHE */
510
511 /* Global lock */
512 static LCK_GRP_DECLARE(mbuf_mlock_grp, "mbuf");
513 static LCK_MTX_DECLARE(mbuf_mlock_data, &mbuf_mlock_grp);
514 static lck_mtx_t *const mbuf_mlock = &mbuf_mlock_data;
515
516 #if CONFIG_MBUF_MCACHE
517 /* Back-end (common) layer */
518 static uint64_t mb_expand_cnt;
519 static uint64_t mb_expand_cl_cnt;
520 static uint64_t mb_expand_cl_total;
521 static uint64_t mb_expand_bigcl_cnt;
522 static uint64_t mb_expand_bigcl_total;
523 static uint64_t mb_expand_16kcl_cnt;
524 static uint64_t mb_expand_16kcl_total;
525 static boolean_t mbuf_worker_needs_wakeup; /* wait channel for mbuf worker */
526 static uint32_t mbuf_worker_run_cnt;
527 static uint64_t mbuf_worker_last_runtime;
528 static uint64_t mbuf_drain_last_runtime;
529 static int mbuf_worker_ready; /* worker thread is runnable */
530 static unsigned int ncpu; /* number of CPUs */
531 static ppnum_t *mcl_paddr; /* Array of cluster physical addresses */
532 static ppnum_t mcl_pages; /* Size of array (# physical pages) */
533 static ppnum_t mcl_paddr_base; /* Handle returned by IOMapper::iovmAlloc() */
534 static mcache_t *ref_cache; /* Cache of cluster reference & flags */
535 static mcache_t *mcl_audit_con_cache; /* Audit contents cache */
536 unsigned int mbuf_debug; /* patchable mbuf mcache flags */
537 #endif /* CONFIG_MBUF_DEBUG */
538 static unsigned int mb_normalized; /* number of packets "normalized" */
539
540 #define MB_GROWTH_AGGRESSIVE 1 /* Threshold: 1/2 of total */
541 #define MB_GROWTH_NORMAL 2 /* Threshold: 3/4 of total */
542
543 typedef enum {
544 MC_MBUF = 0, /* Regular mbuf */
545 MC_CL, /* Cluster */
546 MC_BIGCL, /* Large (4KB) cluster */
547 MC_16KCL, /* Jumbo (16KB) cluster */
548 MC_MBUF_CL, /* mbuf + cluster */
549 MC_MBUF_BIGCL, /* mbuf + large (4KB) cluster */
550 MC_MBUF_16KCL /* mbuf + jumbo (16KB) cluster */
551 } mbuf_class_t;
552
553 #define MBUF_CLASS_MIN MC_MBUF
554 #define MBUF_CLASS_MAX MC_MBUF_16KCL
555 #define MBUF_CLASS_LAST MC_16KCL
556 #define MBUF_CLASS_VALID(c) \
557 ((int)(c) >= MBUF_CLASS_MIN && (int)(c) <= MBUF_CLASS_MAX)
558 #define MBUF_CLASS_COMPOSITE(c) \
559 ((int)(c) > MBUF_CLASS_LAST)
560
561
562 /*
563 * mbuf specific mcache allocation request flags.
564 */
565 #define MCR_COMP MCR_USR1 /* for MC_MBUF_{CL,BIGCL,16KCL} caches */
566
567 /*
568 * Per-cluster slab structure.
569 *
570 * A slab is a cluster control structure that contains one or more object
571 * chunks; the available chunks are chained in the slab's freelist (sl_head).
572 * Each time a chunk is taken out of the slab, the slab's reference count
573 * gets incremented. When all chunks have been taken out, the empty slab
574 * gets removed (SLF_DETACHED) from the class's slab list. A chunk that is
575 * returned to a slab causes the slab's reference count to be decremented;
576 * it also causes the slab to be reinserted back to class's slab list, if
577 * it's not already done.
578 *
579 * Compartmentalizing of the object chunks into slabs allows us to easily
580 * merge one or more slabs together when the adjacent slabs are idle, as
581 * well as to convert or move a slab from one class to another; e.g. the
582 * mbuf cluster slab can be converted to a regular cluster slab when all
583 * mbufs in the slab have been freed.
584 *
585 * A slab may also span across multiple clusters for chunks larger than
586 * a cluster's size. In this case, only the slab of the first cluster is
587 * used. The rest of the slabs are marked with SLF_PARTIAL to indicate
588 * that they are part of the larger slab.
589 *
590 * Each slab controls a page of memory.
591 */
592 typedef struct mcl_slab {
593 struct mcl_slab *sl_next; /* neighboring slab */
594 u_int8_t sl_class; /* controlling mbuf class */
595 int8_t sl_refcnt; /* outstanding allocations */
596 int8_t sl_chunks; /* chunks (bufs) in this slab */
597 u_int16_t sl_flags; /* slab flags (see below) */
598 u_int16_t sl_len; /* slab length */
599 void *sl_base; /* base of allocated memory */
600 void *sl_head; /* first free buffer */
601 TAILQ_ENTRY(mcl_slab) sl_link; /* next/prev slab on freelist */
602 } mcl_slab_t;
603
604 #define SLF_MAPPED 0x0001 /* backed by a mapped page */
605 #define SLF_PARTIAL 0x0002 /* part of another slab */
606 #define SLF_DETACHED 0x0004 /* not in slab freelist */
607
608 /*
609 * The array of slabs are broken into groups of arrays per 1MB of kernel
610 * memory to reduce the footprint. Each group is allocated on demand
611 * whenever a new piece of memory mapped in from the VM crosses the 1MB
612 * boundary.
613 */
614 #define NSLABSPMB ((1 << MBSHIFT) >> PAGE_SHIFT)
615
616 typedef struct mcl_slabg {
617 mcl_slab_t *slg_slab; /* group of slabs */
618 } mcl_slabg_t;
619
620 /*
621 * Number of slabs needed to control a 16KB cluster object.
622 */
623 #define NSLABSP16KB (M16KCLBYTES >> PAGE_SHIFT)
624
625 #if CONFIG_MBUF_MCACHE
626 /*
627 * Per-cluster audit structure.
628 */
629 typedef struct {
630 mcache_audit_t **cl_audit; /* array of audits */
631 } mcl_audit_t;
632
633 typedef struct {
634 struct thread *msa_thread; /* thread doing transaction */
635 struct thread *msa_pthread; /* previous transaction thread */
636 uint32_t msa_tstamp; /* transaction timestamp (ms) */
637 uint32_t msa_ptstamp; /* prev transaction timestamp (ms) */
638 uint16_t msa_depth; /* pc stack depth */
639 uint16_t msa_pdepth; /* previous transaction pc stack */
640 void *msa_stack[MCACHE_STACK_DEPTH];
641 void *msa_pstack[MCACHE_STACK_DEPTH];
642 } mcl_scratch_audit_t;
643
644 typedef struct {
645 /*
646 * Size of data from the beginning of an mbuf that covers m_hdr,
647 * pkthdr and m_ext structures. If auditing is enabled, we allocate
648 * a shadow mbuf structure of this size inside each audit structure,
649 * and the contents of the real mbuf gets copied into it when the mbuf
650 * is freed. This allows us to pattern-fill the mbuf for integrity
651 * check, and to preserve any constructed mbuf fields (e.g. mbuf +
652 * cluster cache case). Note that we don't save the contents of
653 * clusters when they are freed; we simply pattern-fill them.
654 */
655 u_int8_t sc_mbuf[(_MSIZE - _MHLEN) + sizeof(_m_ext_t)];
656 mcl_scratch_audit_t sc_scratch __attribute__((aligned(8)));
657 } mcl_saved_contents_t;
658
659 #define AUDIT_CONTENTS_SIZE (sizeof (mcl_saved_contents_t))
660
661 #define MCA_SAVED_MBUF_PTR(_mca) \
662 ((struct mbuf *)(void *)((mcl_saved_contents_t *) \
663 (_mca)->mca_contents)->sc_mbuf)
664 #define MCA_SAVED_MBUF_SIZE \
665 (sizeof (((mcl_saved_contents_t *)0)->sc_mbuf))
666 #define MCA_SAVED_SCRATCH_PTR(_mca) \
667 (&((mcl_saved_contents_t *)(_mca)->mca_contents)->sc_scratch)
668
669 /*
670 * mbuf specific mcache audit flags
671 */
672 #define MB_INUSE 0x01 /* object has not been returned to slab */
673 #define MB_COMP_INUSE 0x02 /* object has not been returned to cslab */
674 #define MB_SCVALID 0x04 /* object has valid saved contents */
675
676 /*
677 * Each of the following two arrays hold up to nmbclusters elements.
678 */
679 static mcl_audit_t *mclaudit; /* array of cluster audit information */
680 static unsigned int maxclaudit; /* max # of entries in audit table */
681 static mcl_slabg_t **slabstbl; /* cluster slabs table */
682 static unsigned int maxslabgrp; /* max # of entries in slabs table */
683 static unsigned int slabgrp; /* # of entries in slabs table */
684 #endif /* CONFIG_MBUF_MCACHE */
685
686 /* Globals */
687 int nclusters; /* # of clusters for non-jumbo (legacy) sizes */
688 int njcl; /* # of clusters for jumbo sizes */
689 int njclbytes; /* size of a jumbo cluster */
690 unsigned char *mbutl; /* first mapped cluster address */
691 unsigned char *embutl; /* ending virtual address of mclusters */
692 int max_linkhdr; /* largest link-level header */
693 int max_protohdr; /* largest protocol header */
694 int max_hdr; /* largest link+protocol header */
695 int max_datalen; /* MHLEN - max_hdr */
696
697 #if CONFIG_MBUF_MCACHE
698 static boolean_t mclverify; /* debug: pattern-checking */
699 static boolean_t mcltrace; /* debug: stack tracing */
700 static boolean_t mclfindleak; /* debug: leak detection */
701 static boolean_t mclexpleak; /* debug: expose leak info to user space */
702
703 static struct timeval mb_start; /* beginning of time */
704
705 /* mbuf leak detection variables */
706 static struct mleak_table mleak_table;
707 static mleak_stat_t *mleak_stat;
708
709 #define MLEAK_STAT_SIZE(n) \
710 __builtin_offsetof(mleak_stat_t, ml_trace[n])
711
712 struct mallocation {
713 mcache_obj_t *element; /* the alloc'ed element, NULL if unused */
714 u_int32_t trace_index; /* mtrace index for corresponding backtrace */
715 u_int32_t count; /* How many objects were requested */
716 u_int64_t hitcount; /* for determining hash effectiveness */
717 };
718
719 struct mtrace {
720 u_int64_t collisions;
721 u_int64_t hitcount;
722 u_int64_t allocs;
723 u_int64_t depth;
724 uintptr_t addr[MLEAK_STACK_DEPTH];
725 };
726
727 /* Size must be a power of two for the zhash to be able to just mask off bits */
728 #define MLEAK_ALLOCATION_MAP_NUM 512
729 #define MLEAK_TRACE_MAP_NUM 256
730
731 /*
732 * Sample factor for how often to record a trace. This is overwritable
733 * by the boot-arg mleak_sample_factor.
734 */
735 #define MLEAK_SAMPLE_FACTOR 500
736
737 /*
738 * Number of top leakers recorded.
739 */
740 #define MLEAK_NUM_TRACES 5
741
742 #define MB_LEAK_SPACING_64 " "
743 #define MB_LEAK_SPACING_32 " "
744
745
746 #define MB_LEAK_HDR_32 "\n\
747 trace [1] trace [2] trace [3] trace [4] trace [5] \n\
748 ---------- ---------- ---------- ---------- ---------- \n\
749 "
750
751 #define MB_LEAK_HDR_64 "\n\
752 trace [1] trace [2] trace [3] \
753 trace [4] trace [5] \n\
754 ------------------ ------------------ ------------------ \
755 ------------------ ------------------ \n\
756 "
757
758 static uint32_t mleak_alloc_buckets = MLEAK_ALLOCATION_MAP_NUM;
759 static uint32_t mleak_trace_buckets = MLEAK_TRACE_MAP_NUM;
760
761 /* Hashmaps of allocations and their corresponding traces */
762 static struct mallocation *mleak_allocations;
763 static struct mtrace *mleak_traces;
764 static struct mtrace *mleak_top_trace[MLEAK_NUM_TRACES];
765
766 /* Lock to protect mleak tables from concurrent modification */
767 static LCK_GRP_DECLARE(mleak_lock_grp, "mleak_lock");
768 static LCK_MTX_DECLARE(mleak_lock_data, &mleak_lock_grp);
769 static lck_mtx_t *const mleak_lock = &mleak_lock_data;
770
771 /* *Failed* large allocations. */
772 struct mtracelarge {
773 uint64_t size;
774 uint64_t depth;
775 uintptr_t addr[MLEAK_STACK_DEPTH];
776 };
777
778 #define MTRACELARGE_NUM_TRACES 5
779 static struct mtracelarge mtracelarge_table[MTRACELARGE_NUM_TRACES];
780
781 static void mtracelarge_register(size_t size);
782 #endif /* CONFIG_MBUF_MCACHE */
783
784 /* Lock to protect the completion callback table */
785 static LCK_GRP_DECLARE(mbuf_tx_compl_tbl_lck_grp, "mbuf_tx_compl_tbl");
786 LCK_RW_DECLARE(mbuf_tx_compl_tbl_lock, &mbuf_tx_compl_tbl_lck_grp);
787
788 extern u_int32_t high_sb_max;
789
790 /* The minimum number of objects that are allocated, to start. */
791 #define MINCL 32
792 #define MINBIGCL (MINCL >> 1)
793 #define MIN16KCL (MINCL >> 2)
794
795 /* Low watermarks (only map in pages once free counts go below) */
796 #define MBIGCL_LOWAT MINBIGCL
797 #define M16KCL_LOWAT MIN16KCL
798
799 typedef struct {
800 mbuf_class_t mtbl_class; /* class type */
801 #if CONFIG_MBUF_MCACHE
802 mcache_t *mtbl_cache; /* mcache for this buffer class */
803 TAILQ_HEAD(mcl_slhead, mcl_slab) mtbl_slablist; /* slab list */
804 mcache_obj_t *mtbl_cobjlist; /* composite objects freelist */
805 #endif /* CONFIG_MBUF_MCACHE */
806 mb_class_stat_t *mtbl_stats; /* statistics fetchable via sysctl */
807 u_int32_t mtbl_maxsize; /* maximum buffer size */
808 int mtbl_minlimit; /* minimum allowed */
809 int mtbl_maxlimit; /* maximum allowed */
810 u_int32_t mtbl_wantpurge; /* purge during next reclaim */
811 uint32_t mtbl_avgtotal; /* average total on iOS */
812 u_int32_t mtbl_expand; /* worker should expand the class */
813 } mbuf_table_t;
814
815 #define m_class(c) mbuf_table[c].mtbl_class
816 #if CONFIG_MBUF_MCACHE
817 #define m_cache(c) mbuf_table[c].mtbl_cache
818 #define m_slablist(c) mbuf_table[c].mtbl_slablist
819 #define m_cobjlist(c) mbuf_table[c].mtbl_cobjlist
820 #else
821 #define m_stats(c) mbuf_table[c].mtbl_stats
822 #endif /* CONFIG_MBUF_MCACHE */
823 #define m_maxsize(c) mbuf_table[c].mtbl_maxsize
824 #define m_minlimit(c) mbuf_table[c].mtbl_minlimit
825 #define m_maxlimit(c) mbuf_table[c].mtbl_maxlimit
826 #define m_wantpurge(c) mbuf_table[c].mtbl_wantpurge
827 #define m_cname(c) mbuf_table[c].mtbl_stats->mbcl_cname
828 #define m_size(c) mbuf_table[c].mtbl_stats->mbcl_size
829 #define m_total(c) mbuf_table[c].mtbl_stats->mbcl_total
830 #define m_active(c) mbuf_table[c].mtbl_stats->mbcl_active
831 #define m_infree(c) mbuf_table[c].mtbl_stats->mbcl_infree
832 #define m_slab_cnt(c) mbuf_table[c].mtbl_stats->mbcl_slab_cnt
833 #define m_alloc_cnt(c) mbuf_table[c].mtbl_stats->mbcl_alloc_cnt
834 #define m_free_cnt(c) mbuf_table[c].mtbl_stats->mbcl_free_cnt
835 #define m_notified(c) mbuf_table[c].mtbl_stats->mbcl_notified
836 #define m_purge_cnt(c) mbuf_table[c].mtbl_stats->mbcl_purge_cnt
837 #define m_fail_cnt(c) mbuf_table[c].mtbl_stats->mbcl_fail_cnt
838 #define m_ctotal(c) mbuf_table[c].mtbl_stats->mbcl_ctotal
839 #define m_release_cnt(c) mbuf_table[c].mtbl_stats->mbcl_release_cnt
840 #define m_region_expand(c) mbuf_table[c].mtbl_expand
841
842 static mbuf_table_t mbuf_table[] = {
843 #if CONFIG_MBUF_MCACHE
844 /*
845 * The caches for mbufs, regular clusters and big clusters.
846 * The average total values were based on data gathered by actual
847 * usage patterns on iOS.
848 */
849 { MC_MBUF, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_MBUF)),
850 NULL, NULL, 0, 0, 0, 0, 3000, 0 },
851 { MC_CL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_CL)),
852 NULL, NULL, 0, 0, 0, 0, 2000, 0 },
853 { MC_BIGCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_BIGCL)),
854 NULL, NULL, 0, 0, 0, 0, 1000, 0 },
855 { MC_16KCL, NULL, TAILQ_HEAD_INITIALIZER(m_slablist(MC_16KCL)),
856 NULL, NULL, 0, 0, 0, 0, 200, 0 },
857 /*
858 * The following are special caches; they serve as intermediate
859 * caches backed by the above rudimentary caches. Each object
860 * in the cache is an mbuf with a cluster attached to it. Unlike
861 * the above caches, these intermediate caches do not directly
862 * deal with the slab structures; instead, the constructed
863 * cached elements are simply stored in the freelists.
864 */
865 { MC_MBUF_CL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 2000, 0 },
866 { MC_MBUF_BIGCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 1000, 0 },
867 { MC_MBUF_16KCL, NULL, { NULL, NULL }, NULL, NULL, 0, 0, 0, 0, 200, 0 },
868 #else
869 { .mtbl_class = MC_MBUF },
870 { .mtbl_class = MC_CL },
871 { .mtbl_class = MC_BIGCL },
872 { .mtbl_class = MC_16KCL },
873 { .mtbl_class = MC_MBUF_CL },
874 { .mtbl_class = MC_MBUF_BIGCL },
875 { .mtbl_class = MC_MBUF_16KCL },
876 #endif /* CONFIG_MBUF_MCACHE */
877 };
878
879 #define NELEM(a) (sizeof (a) / sizeof ((a)[0]))
880
881 #if SKYWALK && CONFIG_MBUF_MCACHE
882 #define MC_THRESHOLD_SCALE_DOWN_FACTOR 2
883 static unsigned int mc_threshold_scale_down_factor =
884 MC_THRESHOLD_SCALE_DOWN_FACTOR;
885 #endif /* SKYWALK */
886
887 #if CONFIG_MBUF_MCACHE
888 static uint32_t
m_avgtotal(mbuf_class_t c)889 m_avgtotal(mbuf_class_t c)
890 {
891 #if SKYWALK
892 return if_is_fsw_transport_netagent_enabled() ?
893 (mbuf_table[c].mtbl_avgtotal / mc_threshold_scale_down_factor) :
894 mbuf_table[c].mtbl_avgtotal;
895 #else /* !SKYWALK */
896 return mbuf_table[c].mtbl_avgtotal;
897 #endif /* SKYWALK */
898 }
899 #endif /* CONFIG_MBUF_MCACHE */
900
901 #if CONFIG_MBUF_MCACHE
902 static void *mb_waitchan = &mbuf_table; /* wait channel for all caches */
903 static int mb_waiters; /* number of waiters */
904 #endif /* CONFIG_MBUF_MCACHE */
905
906 #define MB_WDT_MAXTIME 10 /* # of secs before watchdog panic */
907 #if CONFIG_MBUF_MCACHE
908 static struct timeval mb_wdtstart; /* watchdog start timestamp */
909 static char *mbuf_dump_buf;
910
911 #define MBUF_DUMP_BUF_SIZE 4096
912
913 /*
914 * mbuf watchdog is enabled by default. It is also toggeable via the
915 * kern.ipc.mb_watchdog sysctl.
916 * Garbage collection is enabled by default on embedded platforms.
917 * mb_drain_maxint controls the amount of time to wait (in seconds) before
918 * consecutive calls to mbuf_drain().
919 */
920 static unsigned int mb_watchdog = 1;
921 #if !XNU_TARGET_OS_OSX
922 static unsigned int mb_drain_maxint = 60;
923 #else /* XNU_TARGET_OS_OSX */
924 static unsigned int mb_drain_maxint = 0;
925 #endif /* XNU_TARGET_OS_OSX */
926 #endif /* CONFIG_MBUF_MCACHE */
927 static unsigned int mb_memory_pressure_percentage = 80;
928
929 uintptr_t mb_obscure_extfree __attribute__((visibility("hidden")));
930 uintptr_t mb_obscure_extref __attribute__((visibility("hidden")));
931
932 /* Red zone */
933 static u_int32_t mb_redzone_cookie;
934 static void m_redzone_init(struct mbuf *);
935 static void m_redzone_verify(struct mbuf *m);
936
937 static void m_set_rfa(struct mbuf *, struct ext_ref *);
938
939 #if CONFIG_MBUF_MCACHE
940 /* The following are used to serialize m_clalloc() */
941 static boolean_t mb_clalloc_busy;
942 static void *mb_clalloc_waitchan = &mb_clalloc_busy;
943 static int mb_clalloc_waiters;
944 #endif /* CONFIG_MBUF_MCACHE */
945
946 static void mbuf_mtypes_sync(boolean_t);
947 static int mbstat_sysctl SYSCTL_HANDLER_ARGS;
948 static void mbuf_stat_sync(void);
949 static int mb_stat_sysctl SYSCTL_HANDLER_ARGS;
950 #if CONFIG_MBUF_MCACHE
951 static int mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS;
952 static int mleak_table_sysctl SYSCTL_HANDLER_ARGS;
953 static char *mbuf_dump(void);
954 #endif /* CONFIG_MBUF_MCACHE */
955 static void mbuf_table_init(void);
956 static inline void m_incref(struct mbuf *);
957 static inline u_int16_t m_decref(struct mbuf *);
958 static void mbuf_watchdog_defunct(thread_call_param_t, thread_call_param_t);
959 #if CONFIG_MBUF_MCACHE
960 static int m_clalloc(const u_int32_t, const int, const u_int32_t);
961 static void mbuf_worker_thread_init(void);
962 static mcache_obj_t *slab_alloc(mbuf_class_t, int);
963 static void slab_free(mbuf_class_t, mcache_obj_t *);
964 static unsigned int mbuf_slab_alloc(void *, mcache_obj_t ***,
965 unsigned int, int);
966 static void mbuf_slab_free(void *, mcache_obj_t *, int);
967 static void mbuf_slab_audit(void *, mcache_obj_t *, boolean_t);
968 static void mbuf_slab_notify(void *, u_int32_t);
969 static unsigned int cslab_alloc(mbuf_class_t, mcache_obj_t ***,
970 unsigned int);
971 static unsigned int cslab_free(mbuf_class_t, mcache_obj_t *, int);
972 static unsigned int mbuf_cslab_alloc(void *, mcache_obj_t ***,
973 unsigned int, int);
974 static void mbuf_cslab_free(void *, mcache_obj_t *, int);
975 static void mbuf_cslab_audit(void *, mcache_obj_t *, boolean_t);
976 static int freelist_populate(mbuf_class_t, unsigned int, int);
977 static void freelist_init(mbuf_class_t);
978 static boolean_t mbuf_cached_above(mbuf_class_t, int);
979 static boolean_t mbuf_steal(mbuf_class_t, unsigned int);
980 static void m_reclaim(mbuf_class_t, unsigned int, boolean_t);
981 static int m_howmany(int, size_t);
982 static void mbuf_worker_thread(void);
983 static void mbuf_watchdog(void);
984 static boolean_t mbuf_sleep(mbuf_class_t, unsigned int, int);
985
986 static void mcl_audit_init(void *, mcache_audit_t **, mcache_obj_t **,
987 size_t, unsigned int);
988 static void mcl_audit_free(void *, unsigned int);
989 static mcache_audit_t *mcl_audit_buf2mca(mbuf_class_t, mcache_obj_t *);
990 static void mcl_audit_mbuf(mcache_audit_t *, void *, boolean_t, boolean_t);
991 static void mcl_audit_cluster(mcache_audit_t *, void *, size_t, boolean_t,
992 boolean_t);
993 static void mcl_audit_restore_mbuf(struct mbuf *, mcache_audit_t *, boolean_t);
994 static void mcl_audit_save_mbuf(struct mbuf *, mcache_audit_t *);
995 static void mcl_audit_scratch(mcache_audit_t *);
996 static void mcl_audit_mcheck_panic(struct mbuf *);
997 static void mcl_audit_verify_nextptr(void *, mcache_audit_t *);
998
999 static void mleak_activate(void);
1000 static void mleak_logger(u_int32_t, mcache_obj_t *, boolean_t);
1001 static boolean_t mleak_log(uintptr_t *, mcache_obj_t *, uint32_t, int);
1002 static void mleak_free(mcache_obj_t *);
1003 static void mleak_sort_traces(void);
1004 static void mleak_update_stats(void);
1005
1006 static mcl_slab_t *slab_get(void *);
1007 static void slab_init(mcl_slab_t *, mbuf_class_t, u_int32_t,
1008 void *, void *, unsigned int, int, int);
1009 static void slab_insert(mcl_slab_t *, mbuf_class_t);
1010 static void slab_remove(mcl_slab_t *, mbuf_class_t);
1011 static boolean_t slab_inrange(mcl_slab_t *, void *);
1012 static void slab_nextptr_panic(mcl_slab_t *, void *);
1013 static void slab_detach(mcl_slab_t *);
1014 static boolean_t slab_is_detached(mcl_slab_t *);
1015 #else /* !CONFIG_MBUF_MCACHE */
1016 static void mbuf_watchdog_drain_composite(thread_call_param_t, thread_call_param_t);
1017 static struct mbuf *mz_alloc(zalloc_flags_t);
1018 static void mz_free(struct mbuf *);
1019 static struct ext_ref *mz_ref_alloc(zalloc_flags_t);
1020 static void mz_ref_free(struct ext_ref *);
1021 static void *mz_cl_alloc(zone_id_t, zalloc_flags_t);
1022 static void mz_cl_free(zone_id_t, void *);
1023 static struct mbuf *mz_composite_alloc(mbuf_class_t, zalloc_flags_t);
1024 static zstack_t mz_composite_alloc_n(mbuf_class_t, unsigned int, zalloc_flags_t);
1025 static void mz_composite_free(mbuf_class_t, struct mbuf *);
1026 static void mz_composite_free_n(mbuf_class_t, zstack_t);
1027 static void *mz_composite_build(zone_id_t, zalloc_flags_t);
1028 static void *mz_composite_mark_valid(zone_id_t, void *);
1029 static void *mz_composite_mark_invalid(zone_id_t, void *);
1030 static void mz_composite_destroy(zone_id_t, void *);
1031
1032 ZONE_DEFINE_ID(ZONE_ID_MBUF_REF, "mbuf.ref", struct ext_ref,
1033 ZC_CACHING | ZC_NOPGZ | ZC_KASAN_NOQUARANTINE);
1034 ZONE_DEFINE_ID(ZONE_ID_MBUF, "mbuf", struct mbuf,
1035 ZC_CACHING | ZC_NOPGZ | ZC_KASAN_NOQUARANTINE);
1036 ZONE_DEFINE_ID(ZONE_ID_CLUSTER_2K, "mbuf.cluster.2k", union mcluster,
1037 ZC_CACHING | ZC_NOPGZ | ZC_KASAN_NOQUARANTINE | ZC_DATA);
1038 ZONE_DEFINE_ID(ZONE_ID_CLUSTER_4K, "mbuf.cluster.4k", union mbigcluster,
1039 ZC_CACHING | ZC_NOPGZ | ZC_KASAN_NOQUARANTINE | ZC_DATA);
1040 ZONE_DEFINE_ID(ZONE_ID_CLUSTER_16K, "mbuf.cluster.16k", union m16kcluster,
1041 ZC_CACHING | ZC_NOPGZ | ZC_KASAN_NOQUARANTINE | ZC_DATA);
1042 static_assert(sizeof(union mcluster) == MCLBYTES);
1043 static_assert(sizeof(union mbigcluster) == MBIGCLBYTES);
1044 static_assert(sizeof(union m16kcluster) == M16KCLBYTES);
1045
1046 static const struct zone_cache_ops mz_composite_ops = {
1047 .zc_op_alloc = mz_composite_build,
1048 .zc_op_mark_valid = mz_composite_mark_valid,
1049 .zc_op_mark_invalid = mz_composite_mark_invalid,
1050 .zc_op_free = mz_composite_destroy,
1051 };
1052 ZCACHE_DEFINE(ZONE_ID_MBUF_CLUSTER_2K, "mbuf.composite.2k", struct mbuf,
1053 sizeof(struct mbuf) + sizeof(struct ext_ref) + MCLBYTES,
1054 &mz_composite_ops);
1055 ZCACHE_DEFINE(ZONE_ID_MBUF_CLUSTER_4K, "mbuf.composite.4k", struct mbuf,
1056 sizeof(struct mbuf) + sizeof(struct ext_ref) + MBIGCLBYTES,
1057 &mz_composite_ops);
1058 ZCACHE_DEFINE(ZONE_ID_MBUF_CLUSTER_16K, "mbuf.composite.16k", struct mbuf,
1059 sizeof(struct mbuf) + sizeof(struct ext_ref) + M16KCLBYTES,
1060 &mz_composite_ops);
1061 static_assert(ZONE_ID_MBUF + MC_MBUF == ZONE_ID_MBUF);
1062 static_assert(ZONE_ID_MBUF + MC_CL == ZONE_ID_CLUSTER_2K);
1063 static_assert(ZONE_ID_MBUF + MC_BIGCL == ZONE_ID_CLUSTER_4K);
1064 static_assert(ZONE_ID_MBUF + MC_16KCL == ZONE_ID_CLUSTER_16K);
1065 static_assert(ZONE_ID_MBUF + MC_MBUF_CL == ZONE_ID_MBUF_CLUSTER_2K);
1066 static_assert(ZONE_ID_MBUF + MC_MBUF_BIGCL == ZONE_ID_MBUF_CLUSTER_4K);
1067 static_assert(ZONE_ID_MBUF + MC_MBUF_16KCL == ZONE_ID_MBUF_CLUSTER_16K);
1068
1069 /* Converts a an mbuf class to a zalloc zone ID. */
1070 __attribute__((always_inline))
1071 static inline zone_id_t
m_class_to_zid(mbuf_class_t class)1072 m_class_to_zid(mbuf_class_t class)
1073 {
1074 return ZONE_ID_MBUF + class - MC_MBUF;
1075 }
1076
1077 __attribute__((always_inline))
1078 static inline mbuf_class_t
m_class_from_zid(zone_id_t zid)1079 m_class_from_zid(zone_id_t zid)
1080 {
1081 return MC_MBUF + zid - ZONE_ID_MBUF;
1082 }
1083
1084 static thread_call_t mbuf_defunct_tcall;
1085 static thread_call_t mbuf_drain_tcall;
1086 #endif /* CONFIG_MBUF_MCACHE */
1087
1088 static int m_copyback0(struct mbuf **, int, int, const void *, int, int);
1089 static struct mbuf *m_split0(struct mbuf *, int, int, int);
1090 #if CONFIG_MBUF_MCACHE && (DEBUG || DEVELOPMENT)
1091 #define mbwdog_logger(fmt, ...) _mbwdog_logger(__func__, __LINE__, fmt, ## __VA_ARGS__)
1092 static void _mbwdog_logger(const char *func, const int line, const char *fmt, ...);
1093 static char *mbwdog_logging;
1094 const unsigned mbwdog_logging_size = 4096;
1095 static size_t mbwdog_logging_used;
1096 #else
1097 #define mbwdog_logger(fmt, ...) do { } while (0)
1098 #endif /* CONFIG_MBUF_MCACHE &&DEBUG || DEVELOPMENT */
1099 #if CONFIG_MBUF_MCACHE
1100 static void mbuf_drain_locked(boolean_t);
1101 #endif /* CONFIG_MBUF_MCACHE */
1102
1103 /* flags for m_copyback0 */
1104 #define M_COPYBACK0_COPYBACK 0x0001 /* copyback from cp */
1105 #define M_COPYBACK0_PRESERVE 0x0002 /* preserve original data */
1106 #define M_COPYBACK0_COW 0x0004 /* do copy-on-write */
1107 #define M_COPYBACK0_EXTEND 0x0008 /* extend chain */
1108
1109 /*
1110 * This flag is set for all mbufs that come out of and into the composite
1111 * mbuf + cluster caches, i.e. MC_MBUF_CL and MC_MBUF_BIGCL. mbufs that
1112 * are marked with such a flag have clusters attached to them, and will be
1113 * treated differently when they are freed; instead of being placed back
1114 * into the mbuf and cluster freelists, the composite mbuf + cluster objects
1115 * are placed back into the appropriate composite cache's freelist, and the
1116 * actual freeing is deferred until the composite objects are purged. At
1117 * such a time, this flag will be cleared from the mbufs and the objects
1118 * will be freed into their own separate freelists.
1119 */
1120 #define EXTF_COMPOSITE 0x1
1121
1122 /*
1123 * This flag indicates that the external cluster is read-only, i.e. it is
1124 * or was referred to by more than one mbufs. Once set, this flag is never
1125 * cleared.
1126 */
1127 #define EXTF_READONLY 0x2
1128 /*
1129 * This flag indicates that the external cluster is paired with the mbuf.
1130 * Pairing implies an external free routine defined which will be invoked
1131 * when the reference count drops to the minimum at m_free time. This
1132 * flag is never cleared.
1133 */
1134 #define EXTF_PAIRED 0x4
1135
1136 #define EXTF_MASK \
1137 (EXTF_COMPOSITE | EXTF_READONLY | EXTF_PAIRED)
1138
1139 #define MEXT_MINREF(m) ((m_get_rfa(m))->minref)
1140 #define MEXT_REF(m) ((m_get_rfa(m))->refcnt)
1141 #define MEXT_PREF(m) ((m_get_rfa(m))->prefcnt)
1142 #define MEXT_FLAGS(m) ((m_get_rfa(m))->flags)
1143 #define MEXT_PRIV(m) ((m_get_rfa(m))->priv)
1144 #define MEXT_PMBUF(m) ((m_get_rfa(m))->paired)
1145 #define MEXT_TOKEN(m) ((m_get_rfa(m))->ext_token)
1146 #define MBUF_IS_COMPOSITE(m) \
1147 (MEXT_REF(m) == MEXT_MINREF(m) && \
1148 (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_COMPOSITE)
1149 /*
1150 * This macro can be used to test if the mbuf is paired to an external
1151 * cluster. The test for MEXT_PMBUF being equal to the mbuf in subject
1152 * is important, as EXTF_PAIRED alone is insufficient since it is immutable,
1153 * and thus survives calls to m_free_paired.
1154 */
1155 #define MBUF_IS_PAIRED(m) \
1156 (((m)->m_flags & M_EXT) && \
1157 (MEXT_FLAGS(m) & EXTF_MASK) == EXTF_PAIRED && \
1158 MEXT_PMBUF(m) == (m))
1159
1160 /*
1161 * Macros used to verify the integrity of the mbuf.
1162 */
1163 #if CONFIG_MBUF_MCACHE
1164 #define _MCHECK(m) { \
1165 if ((m)->m_type != MT_FREE && !MBUF_IS_PAIRED(m)) { \
1166 if (mclaudit == NULL) \
1167 panic("MCHECK: m_type=%d m=%p", \
1168 (u_int16_t)(m)->m_type, m); \
1169 else \
1170 mcl_audit_mcheck_panic(m); \
1171 } \
1172 }
1173 #else
1174 #define _MCHECK(m) \
1175 if ((m)->m_type != MT_FREE && !MBUF_IS_PAIRED(m)) { \
1176 panic("MCHECK: m_type=%d m=%p", \
1177 (u_int16_t)(m)->m_type, m); \
1178 }
1179 #endif /* CONFIG_MBUF_MCACHE */
1180
1181 /*
1182 * Macro version of mtod.
1183 */
1184 #define MTOD(m, t) ((t)((m)->m_data))
1185
1186 #if CONFIG_MBUF_MCACHE
1187 #define MBUF_IN_MAP(addr) \
1188 ((unsigned char *)(addr) >= mbutl && \
1189 (unsigned char *)(addr) < embutl)
1190
1191 #define MRANGE(addr) { \
1192 if (!MBUF_IN_MAP(addr)) \
1193 panic("MRANGE: address out of range 0x%p", addr); \
1194 }
1195
1196 /*
1197 * Macros to obtain page index given a base cluster address
1198 */
1199 #define MTOPG(x) (((unsigned char *)x - mbutl) >> PAGE_SHIFT)
1200 #define PGTOM(x) (mbutl + (x << PAGE_SHIFT))
1201
1202 /*
1203 * Macro to find the mbuf index relative to a base.
1204 */
1205 #define MBPAGEIDX(c, m) \
1206 (((unsigned char *)(m) - (unsigned char *)(c)) >> _MSIZESHIFT)
1207
1208 /*
1209 * Same thing for 2KB cluster index.
1210 */
1211 #define CLPAGEIDX(c, m) \
1212 (((unsigned char *)(m) - (unsigned char *)(c)) >> MCLSHIFT)
1213
1214 /*
1215 * Macro to find 4KB cluster index relative to a base
1216 */
1217 #define BCLPAGEIDX(c, m) \
1218 (((unsigned char *)(m) - (unsigned char *)(c)) >> MBIGCLSHIFT)
1219 #endif /* CONFIG_MBUF_MCACHE */
1220
1221 /*
1222 * Macros used during mbuf and cluster initialization.
1223 */
1224 #define MBUF_INIT_PKTHDR(m) { \
1225 (m)->m_pkthdr.rcvif = NULL; \
1226 (m)->m_pkthdr.pkt_hdr = NULL; \
1227 (m)->m_pkthdr.len = 0; \
1228 (m)->m_pkthdr.csum_flags = 0; \
1229 (m)->m_pkthdr.csum_data = 0; \
1230 (m)->m_pkthdr.vlan_tag = 0; \
1231 (m)->m_pkthdr.comp_gencnt = 0; \
1232 (m)->m_pkthdr.pkt_crumbs = 0; \
1233 m_classifier_init(m, 0); \
1234 m_tag_init(m, 1); \
1235 m_scratch_init(m); \
1236 m_redzone_init(m); \
1237 }
1238
1239 #define MBUF_INIT(m, pkthdr, type) { \
1240 _MCHECK(m); \
1241 (m)->m_next = (m)->m_nextpkt = NULL; \
1242 (m)->m_len = 0; \
1243 (m)->m_type = type; \
1244 if ((pkthdr) == 0) { \
1245 (m)->m_data = (uintptr_t)(m)->m_dat; \
1246 (m)->m_flags = 0; \
1247 } else { \
1248 (m)->m_data = (uintptr_t)(m)->m_pktdat; \
1249 (m)->m_flags = M_PKTHDR; \
1250 MBUF_INIT_PKTHDR(m); \
1251 } \
1252 }
1253
1254 #define MEXT_INIT mext_init
1255
1256 #define MBUF_CL_INIT(m, buf, rfa, ref, flag) \
1257 MEXT_INIT(m, buf, m_maxsize(MC_CL), NULL, NULL, rfa, 0, \
1258 ref, 0, flag, 0, NULL)
1259
1260 #define MBUF_BIGCL_INIT(m, buf, rfa, ref, flag) \
1261 MEXT_INIT(m, buf, m_maxsize(MC_BIGCL), m_bigfree, NULL, rfa, 0, \
1262 ref, 0, flag, 0, NULL)
1263
1264 #define MBUF_16KCL_INIT(m, buf, rfa, ref, flag) \
1265 MEXT_INIT(m, buf, m_maxsize(MC_16KCL), m_16kfree, NULL, rfa, 0, \
1266 ref, 0, flag, 0, NULL)
1267
1268 /*
1269 * Macro to convert BSD malloc sleep flag to mcache's
1270 */
1271 #define MSLEEPF(f) ((!((f) & M_DONTWAIT)) ? MCR_SLEEP : MCR_NOSLEEP)
1272
1273 /*
1274 * The structure that holds all mbuf class statistics exportable via sysctl.
1275 * Similar to mbstat structure, the mb_stat structure is protected by the
1276 * global mbuf lock. It contains additional information about the classes
1277 * that allows for a more accurate view of the state of the allocator.
1278 */
1279 struct mb_stat *mb_stat;
1280 struct omb_stat *omb_stat; /* For backwards compatibility */
1281
1282 #define MB_STAT_SIZE(n) \
1283 __builtin_offsetof(mb_stat_t, mbs_class[n])
1284 #define OMB_STAT_SIZE(n) \
1285 __builtin_offsetof(struct omb_stat, mbs_class[n])
1286
1287 /*
1288 * The legacy structure holding all of the mbuf allocation statistics.
1289 * The actual statistics used by the kernel are stored in the mbuf_table
1290 * instead, and are updated atomically while the global mbuf lock is held.
1291 * They are mirrored in mbstat to support legacy applications (e.g. netstat).
1292 * Unlike before, the kernel no longer relies on the contents of mbstat for
1293 * its operations (e.g. cluster expansion) because the structure is exposed
1294 * to outside and could possibly be modified, therefore making it unsafe.
1295 * With the exception of the mbstat.m_mtypes array (see below), all of the
1296 * statistics are updated as they change.
1297 */
1298 struct mbstat mbstat;
1299
1300 #define MBSTAT_MTYPES_MAX \
1301 (sizeof (mbstat.m_mtypes) / sizeof (mbstat.m_mtypes[0]))
1302
1303 /*
1304 * Allocation statistics related to mbuf types (up to MT_MAX-1) are updated
1305 * atomically and stored in a per-CPU structure which is lock-free; this is
1306 * done in order to avoid writing to the global mbstat data structure which
1307 * would cause false sharing. During sysctl request for kern.ipc.mbstat,
1308 * the statistics across all CPUs will be converged into the mbstat.m_mtypes
1309 * array and returned to the application. Any updates for types greater or
1310 * equal than MT_MAX would be done atomically to the mbstat; this slows down
1311 * performance but is okay since the kernel uses only up to MT_MAX-1 while
1312 * anything beyond that (up to type 255) is considered a corner case.
1313 */
1314 typedef struct {
1315 unsigned int cpu_mtypes[MT_MAX];
1316 } mbuf_mtypes_t;
1317
1318 static mbuf_mtypes_t PERCPU_DATA(mbuf_mtypes);
1319
1320 #define mtype_stat_add(type, n) { \
1321 if ((unsigned)(type) < MT_MAX) { \
1322 mbuf_mtypes_t *mbs = PERCPU_GET(mbuf_mtypes); \
1323 os_atomic_add(&mbs->cpu_mtypes[type], n, relaxed); \
1324 } else if ((unsigned)(type) < (unsigned)MBSTAT_MTYPES_MAX) { \
1325 os_atomic_add((int16_t *)&mbstat.m_mtypes[type], n, relaxed); \
1326 } \
1327 }
1328
1329 #define mtype_stat_sub(t, n) mtype_stat_add(t, -(n))
1330 #define mtype_stat_inc(t) mtype_stat_add(t, 1)
1331 #define mtype_stat_dec(t) mtype_stat_sub(t, 1)
1332
1333 static inline void
mext_init(struct mbuf * m,void * __sized_by (size)buf,u_int size,m_ext_free_func_t free,caddr_t free_arg,struct ext_ref * rfa,u_int16_t min,u_int16_t ref,u_int16_t pref,u_int16_t flag,u_int32_t priv,struct mbuf * pm)1334 mext_init(struct mbuf *m, void *__sized_by(size)buf, u_int size,
1335 m_ext_free_func_t free, caddr_t free_arg, struct ext_ref *rfa,
1336 u_int16_t min, u_int16_t ref, u_int16_t pref, u_int16_t flag,
1337 u_int32_t priv, struct mbuf *pm)
1338 {
1339 m->m_ext.ext_buf = buf;
1340 m->m_ext.ext_size = size;
1341 m->m_data = (uintptr_t)m->m_ext.ext_buf;
1342 m->m_len = 0;
1343 m->m_flags |= M_EXT;
1344 m_set_ext(m, rfa, free, free_arg);
1345 MEXT_MINREF(m) = min;
1346 MEXT_REF(m) = ref;
1347 MEXT_PREF(m) = pref;
1348 MEXT_FLAGS(m) = flag;
1349 MEXT_PRIV(m) = priv;
1350 MEXT_PMBUF(m) = pm;
1351 }
1352
1353 static void
mbuf_mtypes_sync(boolean_t locked)1354 mbuf_mtypes_sync(boolean_t locked)
1355 {
1356 mbuf_mtypes_t mtc;
1357
1358 if (locked) {
1359 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1360 }
1361
1362 mtc = *PERCPU_GET_MASTER(mbuf_mtypes);
1363 percpu_foreach_secondary(mtype, mbuf_mtypes) {
1364 for (int n = 0; n < MT_MAX; n++) {
1365 mtc.cpu_mtypes[n] += mtype->cpu_mtypes[n];
1366 }
1367 }
1368
1369 if (!locked) {
1370 lck_mtx_lock(mbuf_mlock);
1371 }
1372 for (int n = 0; n < MT_MAX; n++) {
1373 mbstat.m_mtypes[n] = mtc.cpu_mtypes[n];
1374 }
1375 if (!locked) {
1376 lck_mtx_unlock(mbuf_mlock);
1377 }
1378 }
1379
1380 static int
1381 mbstat_sysctl SYSCTL_HANDLER_ARGS
1382 {
1383 #pragma unused(oidp, arg1, arg2)
1384
1385 #if CONFIG_MBUF_MCACHE
1386 mbuf_mtypes_sync(FALSE);
1387 #else
1388 lck_mtx_lock(mbuf_mlock);
1389 mbuf_stat_sync();
1390 mbuf_mtypes_sync(TRUE);
1391 lck_mtx_unlock(mbuf_mlock);
1392 #endif
1393
1394 return SYSCTL_OUT(req, &mbstat, sizeof(mbstat));
1395 }
1396
1397 static void
mbuf_stat_sync(void)1398 mbuf_stat_sync(void)
1399 {
1400 mb_class_stat_t *sp;
1401 #if CONFIG_MBUF_MCACHE
1402 mcache_cpu_t *ccp;
1403 mcache_t *cp;
1404 int k, m, bktsize;
1405 #else
1406 int k;
1407 uint64_t drops = 0;
1408 #endif /* CONFIG_MBUF_MCACHE */
1409
1410
1411 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
1412
1413 #if CONFIG_MBUF_MCACHE
1414 for (k = 0; k < NELEM(mbuf_table); k++) {
1415 cp = m_cache(k);
1416 ccp = &cp->mc_cpu[0];
1417 bktsize = ccp->cc_bktsize;
1418 sp = mbuf_table[k].mtbl_stats;
1419
1420 if (cp->mc_flags & MCF_NOCPUCACHE) {
1421 sp->mbcl_mc_state = MCS_DISABLED;
1422 } else if (cp->mc_purge_cnt > 0) {
1423 sp->mbcl_mc_state = MCS_PURGING;
1424 } else if (bktsize == 0) {
1425 sp->mbcl_mc_state = MCS_OFFLINE;
1426 } else {
1427 sp->mbcl_mc_state = MCS_ONLINE;
1428 }
1429
1430 sp->mbcl_mc_cached = 0;
1431 for (m = 0; m < ncpu; m++) {
1432 ccp = &cp->mc_cpu[m];
1433 if (ccp->cc_objs > 0) {
1434 sp->mbcl_mc_cached += ccp->cc_objs;
1435 }
1436 if (ccp->cc_pobjs > 0) {
1437 sp->mbcl_mc_cached += ccp->cc_pobjs;
1438 }
1439 }
1440 sp->mbcl_mc_cached += (cp->mc_full.bl_total * bktsize);
1441 sp->mbcl_active = sp->mbcl_total - sp->mbcl_mc_cached -
1442 sp->mbcl_infree;
1443
1444 sp->mbcl_mc_waiter_cnt = cp->mc_waiter_cnt;
1445 sp->mbcl_mc_wretry_cnt = cp->mc_wretry_cnt;
1446 sp->mbcl_mc_nwretry_cnt = cp->mc_nwretry_cnt;
1447
1448 /* Calculate total count specific to each class */
1449 sp->mbcl_ctotal = sp->mbcl_total;
1450 switch (m_class(k)) {
1451 case MC_MBUF:
1452 /* Deduct mbufs used in composite caches */
1453 sp->mbcl_ctotal -= (m_total(MC_MBUF_CL) +
1454 m_total(MC_MBUF_BIGCL) - m_total(MC_MBUF_16KCL));
1455 break;
1456
1457 case MC_CL:
1458 /* Deduct clusters used in composite cache */
1459 sp->mbcl_ctotal -= m_total(MC_MBUF_CL);
1460 break;
1461
1462 case MC_BIGCL:
1463 /* Deduct clusters used in composite cache */
1464 sp->mbcl_ctotal -= m_total(MC_MBUF_BIGCL);
1465 break;
1466
1467 case MC_16KCL:
1468 /* Deduct clusters used in composite cache */
1469 sp->mbcl_ctotal -= m_total(MC_MBUF_16KCL);
1470 break;
1471
1472 default:
1473 break;
1474 }
1475 }
1476 #else
1477 for (k = 0; k < NELEM(mbuf_table); k++) {
1478 const zone_id_t zid = m_class_to_zid(m_class(k));
1479 const zone_t zone = zone_by_id(zid);
1480 struct zone_basic_stats stats = {};
1481
1482 sp = m_stats(k);
1483 zone_get_stats(zone, &stats);
1484 drops += stats.zbs_alloc_fail;
1485 sp->mbcl_total = stats.zbs_avail;
1486 sp->mbcl_active = stats.zbs_alloc;
1487 /*
1488 * infree is what mcache considers the freelist (uncached)
1489 * free_cnt contains all the cached/uncached elements
1490 * in a zone.
1491 */
1492 sp->mbcl_infree = stats.zbs_free - stats.zbs_cached;
1493 sp->mbcl_fail_cnt = stats.zbs_alloc_fail;
1494 sp->mbcl_ctotal = sp->mbcl_total;
1495
1496 /* These stats are not available in zalloc. */
1497 sp->mbcl_alloc_cnt = 0;
1498 sp->mbcl_free_cnt = 0;
1499 sp->mbcl_notified = 0;
1500 sp->mbcl_purge_cnt = 0;
1501 sp->mbcl_slab_cnt = 0;
1502 sp->mbcl_release_cnt = 0;
1503
1504 /* zalloc caches are always on. */
1505 sp->mbcl_mc_state = MCS_ONLINE;
1506 sp->mbcl_mc_cached = stats.zbs_cached;
1507 /* These stats are not collected by zalloc. */
1508 sp->mbcl_mc_waiter_cnt = 0;
1509 sp->mbcl_mc_wretry_cnt = 0;
1510 sp->mbcl_mc_nwretry_cnt = 0;
1511 }
1512 /* Deduct clusters used in composite cache */
1513 m_ctotal(MC_MBUF) -= (m_total(MC_MBUF_CL) +
1514 m_total(MC_MBUF_BIGCL) -
1515 m_total(MC_MBUF_16KCL));
1516 m_ctotal(MC_CL) -= m_total(MC_MBUF_CL);
1517 m_ctotal(MC_BIGCL) -= m_total(MC_MBUF_BIGCL);
1518 m_ctotal(MC_16KCL) -= m_total(MC_MBUF_16KCL);
1519
1520 /* Update mbstat. */
1521 mbstat.m_mbufs = m_total(MC_MBUF);
1522 mbstat.m_clusters = m_total(MC_CL);
1523 mbstat.m_clfree = m_infree(MC_CL) + m_infree(MC_MBUF_CL);
1524 mbstat.m_drops = drops;
1525 mbstat.m_bigclusters = m_total(MC_BIGCL);
1526 mbstat.m_bigclfree = m_infree(MC_BIGCL) + m_infree(MC_MBUF_BIGCL);
1527 #endif /* CONFIG_MBUF_MCACHE */
1528 }
1529
1530 static int
1531 mb_stat_sysctl SYSCTL_HANDLER_ARGS
1532 {
1533 #pragma unused(oidp, arg1, arg2)
1534 void *statp;
1535 int k, statsz, proc64 = proc_is64bit(req->p);
1536
1537 lck_mtx_lock(mbuf_mlock);
1538 mbuf_stat_sync();
1539
1540 if (!proc64) {
1541 struct omb_class_stat *oc;
1542 struct mb_class_stat *c;
1543
1544 omb_stat->mbs_cnt = mb_stat->mbs_cnt;
1545 oc = &omb_stat->mbs_class[0];
1546 c = &mb_stat->mbs_class[0];
1547 for (k = 0; k < omb_stat->mbs_cnt; k++, oc++, c++) {
1548 (void) snprintf(oc->mbcl_cname, sizeof(oc->mbcl_cname),
1549 "%s", c->mbcl_cname);
1550 oc->mbcl_size = c->mbcl_size;
1551 oc->mbcl_total = c->mbcl_total;
1552 oc->mbcl_active = c->mbcl_active;
1553 oc->mbcl_infree = c->mbcl_infree;
1554 oc->mbcl_slab_cnt = c->mbcl_slab_cnt;
1555 oc->mbcl_alloc_cnt = c->mbcl_alloc_cnt;
1556 oc->mbcl_free_cnt = c->mbcl_free_cnt;
1557 oc->mbcl_notified = c->mbcl_notified;
1558 oc->mbcl_purge_cnt = c->mbcl_purge_cnt;
1559 oc->mbcl_fail_cnt = c->mbcl_fail_cnt;
1560 oc->mbcl_ctotal = c->mbcl_ctotal;
1561 oc->mbcl_release_cnt = c->mbcl_release_cnt;
1562 oc->mbcl_mc_state = c->mbcl_mc_state;
1563 oc->mbcl_mc_cached = c->mbcl_mc_cached;
1564 oc->mbcl_mc_waiter_cnt = c->mbcl_mc_waiter_cnt;
1565 oc->mbcl_mc_wretry_cnt = c->mbcl_mc_wretry_cnt;
1566 oc->mbcl_mc_nwretry_cnt = c->mbcl_mc_nwretry_cnt;
1567 }
1568 statp = omb_stat;
1569 statsz = OMB_STAT_SIZE(NELEM(mbuf_table));
1570 } else {
1571 statp = mb_stat;
1572 statsz = MB_STAT_SIZE(NELEM(mbuf_table));
1573 }
1574
1575 lck_mtx_unlock(mbuf_mlock);
1576
1577 return SYSCTL_OUT(req, statp, statsz);
1578 }
1579
1580 #if !CONFIG_MBUF_MCACHE
1581 /*
1582 * The following functions are wrappers around mbuf
1583 * allocation for zalloc. They all have the prefix "mz"
1584 * which was chosen to avoid conflicts with the mbuf KPIs.
1585 *
1586 * Z_NOPAGEWAIT is used in place of Z_NOWAIT because
1587 * Z_NOPAGEWAIT maps closer to MCR_TRYHARD. Z_NOWAIT will
1588 * fail immediately if it has to take a mutex and that
1589 * may cause packets to be dropped more frequently.
1590 * In general, the mbuf subsystem can sustain grabbing a mutex
1591 * during "non-blocking" allocation and that's the reason
1592 * why Z_NOPAGEWAIT was chosen.
1593 *
1594 * mbufs are elided (removed all pointers) before they are
1595 * returned to the cache. The exception are composite mbufs which
1596 * are re-initialized on allocation.
1597 */
1598 __attribute__((always_inline))
1599 static inline void
m_elide(struct mbuf * m)1600 m_elide(struct mbuf *m)
1601 {
1602 m->m_next = m->m_nextpkt = NULL;
1603 m->m_data = 0;
1604 memset(&m->m_ext, 0, sizeof(m->m_ext));
1605 m->m_pkthdr.rcvif = NULL;
1606 m->m_pkthdr.pkt_hdr = NULL;
1607 m->m_flags |= M_PKTHDR;
1608 m_tag_init(m, 1);
1609 m->m_pkthdr.pkt_flags = 0;
1610 m_scratch_init(m);
1611 m->m_pkthdr.redzone = 0;
1612 m->m_flags &= ~M_PKTHDR;
1613 }
1614
1615 __attribute__((always_inline))
1616 static inline struct mbuf *
mz_alloc(zalloc_flags_t flags)1617 mz_alloc(zalloc_flags_t flags)
1618 {
1619 if (flags & Z_NOWAIT) {
1620 flags ^= Z_NOWAIT | Z_NOPAGEWAIT;
1621 } else if (!(flags & Z_NOPAGEWAIT)) {
1622 flags |= Z_NOFAIL;
1623 }
1624 return zalloc_id(ZONE_ID_MBUF, flags | Z_NOZZC);
1625 }
1626
1627 __attribute__((always_inline))
1628 static inline zstack_t
mz_alloc_n(uint32_t count,zalloc_flags_t flags)1629 mz_alloc_n(uint32_t count, zalloc_flags_t flags)
1630 {
1631 if (flags & Z_NOWAIT) {
1632 flags ^= Z_NOWAIT | Z_NOPAGEWAIT;
1633 } else if (!(flags & Z_NOPAGEWAIT)) {
1634 flags |= Z_NOFAIL;
1635 }
1636 return zalloc_n(ZONE_ID_MBUF, count, flags | Z_NOZZC);
1637 }
1638
1639 __attribute__((always_inline))
1640 static inline void
mz_free(struct mbuf * m)1641 mz_free(struct mbuf *m)
1642 {
1643 #if KASAN
1644 zone_require(zone_by_id(ZONE_ID_MBUF), m);
1645 #endif
1646 m_elide(m);
1647 zfree_nozero(ZONE_ID_MBUF, m);
1648 }
1649
1650 __attribute__((always_inline))
1651 static inline void
mz_free_n(zstack_t list)1652 mz_free_n(zstack_t list)
1653 {
1654 /* Callers of this function have already elided the mbuf. */
1655 zfree_nozero_n(ZONE_ID_MBUF, list);
1656 }
1657
1658 __attribute__((always_inline))
1659 static inline struct ext_ref *
mz_ref_alloc(zalloc_flags_t flags)1660 mz_ref_alloc(zalloc_flags_t flags)
1661 {
1662 if (flags & Z_NOWAIT) {
1663 flags ^= Z_NOWAIT | Z_NOPAGEWAIT;
1664 }
1665 return zalloc_id(ZONE_ID_MBUF_REF, flags | Z_NOZZC);
1666 }
1667
1668 __attribute__((always_inline))
1669 static inline void
mz_ref_free(struct ext_ref * rfa)1670 mz_ref_free(struct ext_ref *rfa)
1671 {
1672 VERIFY(rfa->minref == rfa->refcnt);
1673 #if KASAN
1674 zone_require(zone_by_id(ZONE_ID_MBUF_REF), rfa);
1675 #endif
1676 zfree_nozero(ZONE_ID_MBUF_REF, rfa);
1677 }
1678
1679 __attribute__((always_inline))
1680 static inline void *
mz_cl_alloc(zone_id_t zid,zalloc_flags_t flags)1681 mz_cl_alloc(zone_id_t zid, zalloc_flags_t flags)
1682 {
1683 if (flags & Z_NOWAIT) {
1684 flags ^= Z_NOWAIT | Z_NOPAGEWAIT;
1685 } else if (!(flags & Z_NOPAGEWAIT)) {
1686 flags |= Z_NOFAIL;
1687 }
1688 return (zalloc_id)(zid, flags | Z_NOZZC);
1689 }
1690
1691 __attribute__((always_inline))
1692 static inline void
mz_cl_free(zone_id_t zid,void * cl)1693 mz_cl_free(zone_id_t zid, void *cl)
1694 {
1695 #if KASAN
1696 zone_require(zone_by_id(zid), cl);
1697 #endif
1698 zfree_nozero(zid, cl);
1699 }
1700
1701 __attribute__((always_inline))
1702 static inline zstack_t
mz_composite_alloc_n(mbuf_class_t class,unsigned int n,zalloc_flags_t flags)1703 mz_composite_alloc_n(mbuf_class_t class, unsigned int n, zalloc_flags_t flags)
1704 {
1705 if (flags & Z_NOWAIT) {
1706 flags ^= Z_NOWAIT | Z_NOPAGEWAIT;
1707 }
1708 return (zcache_alloc_n)(m_class_to_zid(class), n, flags,
1709 &mz_composite_ops);
1710 }
1711
1712 __attribute__((always_inline))
1713 static inline struct mbuf *
mz_composite_alloc(mbuf_class_t class,zalloc_flags_t flags)1714 mz_composite_alloc(mbuf_class_t class, zalloc_flags_t flags)
1715 {
1716 zstack_t list = {};
1717 list = mz_composite_alloc_n(class, 1, flags);
1718 if (!zstack_empty(list)) {
1719 return zstack_pop(&list);
1720 } else {
1721 return NULL;
1722 }
1723 }
1724
1725 __attribute__((always_inline))
1726 static inline void
mz_composite_free_n(mbuf_class_t class,zstack_t list)1727 mz_composite_free_n(mbuf_class_t class, zstack_t list)
1728 {
1729 (zcache_free_n)(m_class_to_zid(class), list, &mz_composite_ops);
1730 }
1731
1732 __attribute__((always_inline))
1733 static inline void
mz_composite_free(mbuf_class_t class,struct mbuf * m)1734 mz_composite_free(mbuf_class_t class, struct mbuf *m)
1735 {
1736 zstack_t list = {};
1737 zstack_push(&list, m);
1738 (zcache_free_n)(m_class_to_zid(class), list, &mz_composite_ops);
1739 }
1740
1741 /* Converts composite zone ID to the cluster zone ID. */
1742 __attribute__((always_inline))
1743 static inline zone_id_t
mz_cl_zid(zone_id_t zid)1744 mz_cl_zid(zone_id_t zid)
1745 {
1746 return ZONE_ID_CLUSTER_2K + zid - ZONE_ID_MBUF_CLUSTER_2K;
1747 }
1748
1749 static void *
mz_composite_build(zone_id_t zid,zalloc_flags_t flags)1750 mz_composite_build(zone_id_t zid, zalloc_flags_t flags)
1751 {
1752 const zone_id_t cl_zid = mz_cl_zid(zid);
1753 struct mbuf *m = NULL;
1754 struct ext_ref *rfa = NULL;
1755 void *cl = NULL;
1756
1757 cl = mz_cl_alloc(cl_zid, flags);
1758 if (__improbable(cl == NULL)) {
1759 goto out;
1760 }
1761 rfa = mz_ref_alloc(flags);
1762 if (__improbable(rfa == NULL)) {
1763 goto out_free_cl;
1764 }
1765 m = mz_alloc(flags);
1766 if (__improbable(m == NULL)) {
1767 goto out_free_rfa;
1768 }
1769 MBUF_INIT(m, 0, MT_FREE);
1770 if (zid == ZONE_ID_MBUF_CLUSTER_2K) {
1771 MBUF_CL_INIT(m, cl, rfa, 0, EXTF_COMPOSITE);
1772 } else if (zid == ZONE_ID_MBUF_CLUSTER_4K) {
1773 MBUF_BIGCL_INIT(m, cl, rfa, 0, EXTF_COMPOSITE);
1774 } else {
1775 MBUF_16KCL_INIT(m, cl, rfa, 0, EXTF_COMPOSITE);
1776 }
1777 VERIFY(m->m_flags == M_EXT);
1778 VERIFY(m_get_rfa(m) != NULL && MBUF_IS_COMPOSITE(m));
1779
1780 return m;
1781 out_free_rfa:
1782 mz_ref_free(rfa);
1783 out_free_cl:
1784 mz_cl_free(cl_zid, cl);
1785 out:
1786 return NULL;
1787 }
1788
1789 static void *
mz_composite_mark_valid(zone_id_t zid,void * p)1790 mz_composite_mark_valid(zone_id_t zid, void *p)
1791 {
1792 struct mbuf *m = p;
1793
1794 m = zcache_mark_valid(zone_by_id(ZONE_ID_MBUF), m);
1795 #if KASAN
1796 struct ext_ref *rfa = m_get_rfa(m);
1797 const zone_id_t cl_zid = mz_cl_zid(zid);
1798 void *cl = m->m_ext.ext_buf;
1799
1800 cl = zcache_mark_valid(zone_by_id(cl_zid), cl);
1801 rfa = zcache_mark_valid(zone_by_id(ZONE_ID_MBUF_REF), rfa);
1802 m->m_data = (uintptr_t)cl;
1803 m->m_ext.ext_buf = cl;
1804 m_set_rfa(m, rfa);
1805 #else
1806 #pragma unused(zid)
1807 #endif
1808 VERIFY(MBUF_IS_COMPOSITE(m));
1809
1810 return m;
1811 }
1812
1813 static void *
mz_composite_mark_invalid(zone_id_t zid,void * p)1814 mz_composite_mark_invalid(zone_id_t zid, void *p)
1815 {
1816 struct mbuf *m = p;
1817
1818 VERIFY(MBUF_IS_COMPOSITE(m));
1819 VERIFY(MEXT_REF(m) == MEXT_MINREF(m));
1820 #if KASAN
1821 struct ext_ref *rfa = m_get_rfa(m);
1822 const zone_id_t cl_zid = mz_cl_zid(zid);
1823 void *cl = m->m_ext.ext_buf;
1824
1825 cl = zcache_mark_invalid(zone_by_id(cl_zid), cl);
1826 rfa = zcache_mark_invalid(zone_by_id(ZONE_ID_MBUF_REF), rfa);
1827 m->m_data = (uintptr_t)cl;
1828 m->m_ext.ext_buf = cl;
1829 m_set_rfa(m, rfa);
1830 #else
1831 #pragma unused(zid)
1832 #endif
1833
1834 return zcache_mark_invalid(zone_by_id(ZONE_ID_MBUF), m);
1835 }
1836
1837 static void
mz_composite_destroy(zone_id_t zid,void * p)1838 mz_composite_destroy(zone_id_t zid, void *p)
1839 {
1840 const zone_id_t cl_zid = mz_cl_zid(zid);
1841 struct ext_ref *rfa = NULL;
1842 struct mbuf *m = p;
1843
1844 VERIFY(MBUF_IS_COMPOSITE(m));
1845
1846 MEXT_MINREF(m) = 0;
1847 MEXT_REF(m) = 0;
1848 MEXT_PREF(m) = 0;
1849 MEXT_FLAGS(m) = 0;
1850 MEXT_PRIV(m) = 0;
1851 MEXT_PMBUF(m) = NULL;
1852 MEXT_TOKEN(m) = 0;
1853
1854 rfa = m_get_rfa(m);
1855 m_set_ext(m, NULL, NULL, NULL);
1856
1857 m->m_type = MT_FREE;
1858 m->m_flags = m->m_len = 0;
1859 m->m_next = m->m_nextpkt = NULL;
1860
1861 mz_cl_free(cl_zid, m->m_ext.ext_buf);
1862 m->m_ext.ext_buf = NULL;
1863 mz_ref_free(rfa);
1864 mz_free(m);
1865 }
1866 #endif /* !CONFIG_MBUF_MCACHE */
1867
1868 #if CONFIG_MBUF_MCACHE
1869 static int
1870 mleak_top_trace_sysctl SYSCTL_HANDLER_ARGS
1871 {
1872 #pragma unused(oidp, arg1, arg2)
1873 int i;
1874
1875 /* Ensure leak tracing turned on */
1876 if (!mclfindleak || !mclexpleak) {
1877 return ENXIO;
1878 }
1879
1880 lck_mtx_lock(mleak_lock);
1881 mleak_update_stats();
1882 i = SYSCTL_OUT(req, mleak_stat, MLEAK_STAT_SIZE(MLEAK_NUM_TRACES));
1883 lck_mtx_unlock(mleak_lock);
1884
1885 return i;
1886 }
1887
1888 static int
1889 mleak_table_sysctl SYSCTL_HANDLER_ARGS
1890 {
1891 #pragma unused(oidp, arg1, arg2)
1892 int i = 0;
1893
1894 /* Ensure leak tracing turned on */
1895 if (!mclfindleak || !mclexpleak) {
1896 return ENXIO;
1897 }
1898
1899 lck_mtx_lock(mleak_lock);
1900 i = SYSCTL_OUT(req, &mleak_table, sizeof(mleak_table));
1901 lck_mtx_unlock(mleak_lock);
1902
1903 return i;
1904 }
1905 #endif /* CONFIG_MBUF_MCACHE */
1906
1907 static inline void
m_incref(struct mbuf * m)1908 m_incref(struct mbuf *m)
1909 {
1910 uint16_t new = os_atomic_inc(&MEXT_REF(m), relaxed);
1911
1912 VERIFY(new != 0);
1913 /*
1914 * If cluster is shared, mark it with (sticky) EXTF_READONLY;
1915 * we don't clear the flag when the refcount goes back to the
1916 * minimum, to simplify code calling m_mclhasreference().
1917 */
1918 if (new > (MEXT_MINREF(m) + 1) && !(MEXT_FLAGS(m) & EXTF_READONLY)) {
1919 os_atomic_or(&MEXT_FLAGS(m), EXTF_READONLY, relaxed);
1920 }
1921 }
1922
1923 static inline uint16_t
m_decref(struct mbuf * m)1924 m_decref(struct mbuf *m)
1925 {
1926 VERIFY(MEXT_REF(m) != 0);
1927
1928 return os_atomic_dec(&MEXT_REF(m), acq_rel);
1929 }
1930
1931 static void
mbuf_table_init(void)1932 mbuf_table_init(void)
1933 {
1934 unsigned int b, c, s;
1935 int m, config_mbuf_jumbo = 0;
1936
1937 omb_stat = zalloc_permanent(OMB_STAT_SIZE(NELEM(mbuf_table)),
1938 ZALIGN(struct omb_stat));
1939
1940 mb_stat = zalloc_permanent(MB_STAT_SIZE(NELEM(mbuf_table)),
1941 ZALIGN(mb_stat_t));
1942
1943 mb_stat->mbs_cnt = NELEM(mbuf_table);
1944 for (m = 0; m < NELEM(mbuf_table); m++) {
1945 mbuf_table[m].mtbl_stats = &mb_stat->mbs_class[m];
1946 }
1947
1948 #if CONFIG_MBUF_JUMBO
1949 config_mbuf_jumbo = 1;
1950 #endif /* CONFIG_MBUF_JUMBO */
1951
1952 if (config_mbuf_jumbo == 1 || PAGE_SIZE == M16KCLBYTES) {
1953 /*
1954 * Set aside 1/3 of the mbuf cluster map for jumbo
1955 * clusters; we do this only on platforms where jumbo
1956 * cluster pool is enabled.
1957 */
1958 njcl = nmbclusters / 3;
1959 njclbytes = M16KCLBYTES;
1960 }
1961
1962 /*
1963 * nclusters holds both the 2KB and 4KB pools, so ensure it's
1964 * a multiple of 4KB clusters.
1965 */
1966 nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPG);
1967 if (njcl > 0) {
1968 /*
1969 * Each jumbo cluster takes 8 2KB clusters, so make
1970 * sure that the pool size is evenly divisible by 8;
1971 * njcl is in 2KB unit, hence treated as such.
1972 */
1973 njcl = P2ROUNDDOWN(nmbclusters - nclusters, NCLPJCL);
1974
1975 /* Update nclusters with rounded down value of njcl */
1976 nclusters = P2ROUNDDOWN(nmbclusters - njcl, NCLPG);
1977 }
1978
1979 /*
1980 * njcl is valid only on platforms with 16KB jumbo clusters or
1981 * with 16KB pages, where it is configured to 1/3 of the pool
1982 * size. On these platforms, the remaining is used for 2KB
1983 * and 4KB clusters. On platforms without 16KB jumbo clusters,
1984 * the entire pool is used for both 2KB and 4KB clusters. A 4KB
1985 * cluster can either be splitted into 16 mbufs, or into 2 2KB
1986 * clusters.
1987 *
1988 * +---+---+------------ ... -----------+------- ... -------+
1989 * | c | b | s | njcl |
1990 * +---+---+------------ ... -----------+------- ... -------+
1991 *
1992 * 1/32th of the shared region is reserved for pure 2KB and 4KB
1993 * clusters (1/64th each.)
1994 */
1995 c = P2ROUNDDOWN((nclusters >> 6), NCLPG); /* in 2KB unit */
1996 b = P2ROUNDDOWN((nclusters >> (6 + NCLPBGSHIFT)), NBCLPG); /* in 4KB unit */
1997 s = nclusters - (c + (b << NCLPBGSHIFT)); /* in 2KB unit */
1998
1999 /*
2000 * 1/64th (c) is reserved for 2KB clusters.
2001 */
2002 m_minlimit(MC_CL) = c;
2003 m_maxlimit(MC_CL) = s + c; /* in 2KB unit */
2004 m_maxsize(MC_CL) = m_size(MC_CL) = MCLBYTES;
2005 snprintf(m_cname(MC_CL), MAX_MBUF_CNAME, "cl");
2006
2007 /*
2008 * Another 1/64th (b) of the map is reserved for 4KB clusters.
2009 * It cannot be turned into 2KB clusters or mbufs.
2010 */
2011 m_minlimit(MC_BIGCL) = b;
2012 m_maxlimit(MC_BIGCL) = (s >> NCLPBGSHIFT) + b; /* in 4KB unit */
2013 m_maxsize(MC_BIGCL) = m_size(MC_BIGCL) = MBIGCLBYTES;
2014 snprintf(m_cname(MC_BIGCL), MAX_MBUF_CNAME, "bigcl");
2015
2016 /*
2017 * The remaining 31/32ths (s) are all-purpose (mbufs, 2KB, or 4KB)
2018 */
2019 m_minlimit(MC_MBUF) = 0;
2020 m_maxlimit(MC_MBUF) = s * NMBPCL; /* in mbuf unit */
2021 m_maxsize(MC_MBUF) = m_size(MC_MBUF) = _MSIZE;
2022 snprintf(m_cname(MC_MBUF), MAX_MBUF_CNAME, "mbuf");
2023
2024 /*
2025 * Set limits for the composite classes.
2026 */
2027 m_minlimit(MC_MBUF_CL) = 0;
2028 m_maxlimit(MC_MBUF_CL) = m_maxlimit(MC_CL);
2029 m_maxsize(MC_MBUF_CL) = MCLBYTES;
2030 m_size(MC_MBUF_CL) = m_size(MC_MBUF) + m_size(MC_CL);
2031 snprintf(m_cname(MC_MBUF_CL), MAX_MBUF_CNAME, "mbuf_cl");
2032
2033 m_minlimit(MC_MBUF_BIGCL) = 0;
2034 m_maxlimit(MC_MBUF_BIGCL) = m_maxlimit(MC_BIGCL);
2035 m_maxsize(MC_MBUF_BIGCL) = MBIGCLBYTES;
2036 m_size(MC_MBUF_BIGCL) = m_size(MC_MBUF) + m_size(MC_BIGCL);
2037 snprintf(m_cname(MC_MBUF_BIGCL), MAX_MBUF_CNAME, "mbuf_bigcl");
2038
2039 /*
2040 * And for jumbo classes.
2041 */
2042 m_minlimit(MC_16KCL) = 0;
2043 m_maxlimit(MC_16KCL) = (njcl >> NCLPJCLSHIFT); /* in 16KB unit */
2044 m_maxsize(MC_16KCL) = m_size(MC_16KCL) = M16KCLBYTES;
2045 snprintf(m_cname(MC_16KCL), MAX_MBUF_CNAME, "16kcl");
2046
2047 m_minlimit(MC_MBUF_16KCL) = 0;
2048 m_maxlimit(MC_MBUF_16KCL) = m_maxlimit(MC_16KCL);
2049 m_maxsize(MC_MBUF_16KCL) = M16KCLBYTES;
2050 m_size(MC_MBUF_16KCL) = m_size(MC_MBUF) + m_size(MC_16KCL);
2051 snprintf(m_cname(MC_MBUF_16KCL), MAX_MBUF_CNAME, "mbuf_16kcl");
2052
2053 /*
2054 * Initialize the legacy mbstat structure.
2055 */
2056 bzero(&mbstat, sizeof(mbstat));
2057 mbstat.m_msize = m_maxsize(MC_MBUF);
2058 mbstat.m_mclbytes = m_maxsize(MC_CL);
2059 mbstat.m_minclsize = MINCLSIZE;
2060 mbstat.m_mlen = MLEN;
2061 mbstat.m_mhlen = MHLEN;
2062 mbstat.m_bigmclbytes = m_maxsize(MC_BIGCL);
2063 }
2064
2065 static int
mbuf_get_class(struct mbuf * m)2066 mbuf_get_class(struct mbuf *m)
2067 {
2068 if (m->m_flags & M_EXT) {
2069 uint32_t composite = (MEXT_FLAGS(m) & EXTF_COMPOSITE);
2070 m_ext_free_func_t m_free_func = m_get_ext_free(m);
2071
2072 if (m_free_func == NULL) {
2073 if (composite) {
2074 return MC_MBUF_CL;
2075 } else {
2076 return MC_CL;
2077 }
2078 } else if (m_free_func == m_bigfree) {
2079 if (composite) {
2080 return MC_MBUF_BIGCL;
2081 } else {
2082 return MC_BIGCL;
2083 }
2084 } else if (m_free_func == m_16kfree) {
2085 if (composite) {
2086 return MC_MBUF_16KCL;
2087 } else {
2088 return MC_16KCL;
2089 }
2090 }
2091 }
2092
2093 return MC_MBUF;
2094 }
2095
2096 bool
mbuf_class_under_pressure(struct mbuf * m)2097 mbuf_class_under_pressure(struct mbuf *m)
2098 {
2099 int mclass = mbuf_get_class(m);
2100
2101 #if CONFIG_MBUF_MCACHE
2102 if (m_total(mclass) - m_infree(mclass) >= (m_maxlimit(mclass) * mb_memory_pressure_percentage) / 100) {
2103 /*
2104 * The above computation does not include the per-CPU cached objects.
2105 * As a fast-path check this is good-enough. But now we do
2106 * the "slower" count of the cached objects to know exactly the
2107 * number of active mbufs in use.
2108 *
2109 * We do not take the mbuf_lock here to avoid lock-contention. Numbers
2110 * might be slightly off but we don't try to be 100% accurate.
2111 * At worst, we drop a packet that we shouldn't have dropped or
2112 * we might go slightly above our memory-pressure threshold.
2113 */
2114 mcache_t *cp = m_cache(mclass);
2115 mcache_cpu_t *ccp = &cp->mc_cpu[0];
2116
2117 int bktsize = os_access_once(ccp->cc_bktsize);
2118 uint32_t bl_total = os_access_once(cp->mc_full.bl_total);
2119 uint32_t cached = 0;
2120 int i;
2121
2122 for (i = 0; i < ncpu; i++) {
2123 ccp = &cp->mc_cpu[i];
2124
2125 int cc_objs = os_access_once(ccp->cc_objs);
2126 if (cc_objs > 0) {
2127 cached += cc_objs;
2128 }
2129
2130 int cc_pobjs = os_access_once(ccp->cc_pobjs);
2131 if (cc_pobjs > 0) {
2132 cached += cc_pobjs;
2133 }
2134 }
2135 cached += (bl_total * bktsize);
2136 if (m_total(mclass) - m_infree(mclass) - cached >= (m_maxlimit(mclass) * mb_memory_pressure_percentage) / 100) {
2137 os_log(OS_LOG_DEFAULT,
2138 "%s memory-pressure on mbuf due to class %u, total %u free %u cached %u max %u",
2139 __func__, mclass, m_total(mclass), m_infree(mclass), cached, m_maxlimit(mclass));
2140 return true;
2141 }
2142 }
2143 #else
2144 /*
2145 * Grab the statistics from zalloc.
2146 * We can't call mbuf_stat_sync() since that requires a lock.
2147 */
2148 const zone_id_t zid = m_class_to_zid(m_class(mclass));
2149 const zone_t zone = zone_by_id(zid);
2150 struct zone_basic_stats stats = {};
2151
2152 zone_get_stats(zone, &stats);
2153 if (stats.zbs_avail - stats.zbs_free >= (m_maxlimit(mclass) * mb_memory_pressure_percentage) / 100) {
2154 os_log(OS_LOG_DEFAULT,
2155 "%s memory-pressure on mbuf due to class %u, total %llu free %llu max %u",
2156 __func__, mclass, stats.zbs_avail, stats.zbs_free, m_maxlimit(mclass));
2157 return true;
2158 }
2159 #endif /* CONFIG_MBUF_MCACHE */
2160
2161 return false;
2162 }
2163
2164 #if defined(__LP64__)
2165 typedef struct ncl_tbl {
2166 uint64_t nt_maxmem; /* memory (sane) size */
2167 uint32_t nt_mbpool; /* mbuf pool size */
2168 } ncl_tbl_t;
2169
2170 static const ncl_tbl_t ncl_table[] = {
2171 { (1ULL << GBSHIFT) /* 1 GB */, (64 << MBSHIFT) /* 64 MB */ },
2172 { (1ULL << (GBSHIFT + 2)) /* 4 GB */, (96 << MBSHIFT) /* 96 MB */ },
2173 { (1ULL << (GBSHIFT + 3)) /* 8 GB */, (128 << MBSHIFT) /* 128 MB */ },
2174 { (1ULL << (GBSHIFT + 4)) /* 16 GB */, (256 << MBSHIFT) /* 256 MB */ },
2175 { (1ULL << (GBSHIFT + 5)) /* 32 GB */, (512 << MBSHIFT) /* 512 MB */ },
2176 { 0, 0 }
2177 };
2178 #endif /* __LP64__ */
2179
2180 __private_extern__ unsigned int
mbuf_default_ncl(uint64_t mem)2181 mbuf_default_ncl(uint64_t mem)
2182 {
2183 #if !defined(__LP64__)
2184 unsigned int n;
2185 /*
2186 * 32-bit kernel (default to 64MB of mbuf pool for >= 1GB RAM).
2187 */
2188 if ((n = ((mem / 16) / MCLBYTES)) > 32768) {
2189 n = 32768;
2190 }
2191 #else
2192 unsigned int n, i;
2193 /*
2194 * 64-bit kernel (mbuf pool size based on table).
2195 */
2196 n = ncl_table[0].nt_mbpool;
2197 for (i = 0; ncl_table[i].nt_mbpool != 0; i++) {
2198 if (mem < ncl_table[i].nt_maxmem) {
2199 break;
2200 }
2201 n = ncl_table[i].nt_mbpool;
2202 }
2203 n >>= MCLSHIFT;
2204 #endif /* !__LP64__ */
2205 return n;
2206 }
2207
2208 __private_extern__ void
mbinit(void)2209 mbinit(void)
2210 {
2211 unsigned int m;
2212 #if CONFIG_MBUF_MCACHE
2213 unsigned int initmcl = 0;
2214 thread_t thread = THREAD_NULL;
2215 #endif /* CONFIG_MBUF_MCACHE */
2216
2217 #if CONFIG_MBUF_MCACHE
2218 microuptime(&mb_start);
2219 #endif /* CONFIG_MBUF_MCACHE */
2220
2221 /*
2222 * These MBUF_ values must be equal to their private counterparts.
2223 */
2224 _CASSERT(MBUF_EXT == M_EXT);
2225 _CASSERT(MBUF_PKTHDR == M_PKTHDR);
2226 _CASSERT(MBUF_EOR == M_EOR);
2227 _CASSERT(MBUF_LOOP == M_LOOP);
2228 _CASSERT(MBUF_BCAST == M_BCAST);
2229 _CASSERT(MBUF_MCAST == M_MCAST);
2230 _CASSERT(MBUF_FRAG == M_FRAG);
2231 _CASSERT(MBUF_FIRSTFRAG == M_FIRSTFRAG);
2232 _CASSERT(MBUF_LASTFRAG == M_LASTFRAG);
2233 _CASSERT(MBUF_PROMISC == M_PROMISC);
2234 _CASSERT(MBUF_HASFCS == M_HASFCS);
2235
2236 _CASSERT(MBUF_TYPE_FREE == MT_FREE);
2237 _CASSERT(MBUF_TYPE_DATA == MT_DATA);
2238 _CASSERT(MBUF_TYPE_HEADER == MT_HEADER);
2239 _CASSERT(MBUF_TYPE_SOCKET == MT_SOCKET);
2240 _CASSERT(MBUF_TYPE_PCB == MT_PCB);
2241 _CASSERT(MBUF_TYPE_RTABLE == MT_RTABLE);
2242 _CASSERT(MBUF_TYPE_HTABLE == MT_HTABLE);
2243 _CASSERT(MBUF_TYPE_ATABLE == MT_ATABLE);
2244 _CASSERT(MBUF_TYPE_SONAME == MT_SONAME);
2245 _CASSERT(MBUF_TYPE_SOOPTS == MT_SOOPTS);
2246 _CASSERT(MBUF_TYPE_FTABLE == MT_FTABLE);
2247 _CASSERT(MBUF_TYPE_RIGHTS == MT_RIGHTS);
2248 _CASSERT(MBUF_TYPE_IFADDR == MT_IFADDR);
2249 _CASSERT(MBUF_TYPE_CONTROL == MT_CONTROL);
2250 _CASSERT(MBUF_TYPE_OOBDATA == MT_OOBDATA);
2251
2252 _CASSERT(MBUF_TSO_IPV4 == CSUM_TSO_IPV4);
2253 _CASSERT(MBUF_TSO_IPV6 == CSUM_TSO_IPV6);
2254 _CASSERT(MBUF_CSUM_REQ_SUM16 == CSUM_PARTIAL);
2255 _CASSERT(MBUF_CSUM_TCP_SUM16 == MBUF_CSUM_REQ_SUM16);
2256 _CASSERT(MBUF_CSUM_REQ_ZERO_INVERT == CSUM_ZERO_INVERT);
2257 _CASSERT(MBUF_CSUM_REQ_IP == CSUM_IP);
2258 _CASSERT(MBUF_CSUM_REQ_TCP == CSUM_TCP);
2259 _CASSERT(MBUF_CSUM_REQ_UDP == CSUM_UDP);
2260 _CASSERT(MBUF_CSUM_REQ_TCPIPV6 == CSUM_TCPIPV6);
2261 _CASSERT(MBUF_CSUM_REQ_UDPIPV6 == CSUM_UDPIPV6);
2262 _CASSERT(MBUF_CSUM_DID_IP == CSUM_IP_CHECKED);
2263 _CASSERT(MBUF_CSUM_IP_GOOD == CSUM_IP_VALID);
2264 _CASSERT(MBUF_CSUM_DID_DATA == CSUM_DATA_VALID);
2265 _CASSERT(MBUF_CSUM_PSEUDO_HDR == CSUM_PSEUDO_HDR);
2266
2267 _CASSERT(MBUF_WAITOK == M_WAIT);
2268 _CASSERT(MBUF_DONTWAIT == M_DONTWAIT);
2269 _CASSERT(MBUF_COPYALL == M_COPYALL);
2270
2271 _CASSERT(MBUF_SC2TC(MBUF_SC_BK_SYS) == MBUF_TC_BK);
2272 _CASSERT(MBUF_SC2TC(MBUF_SC_BK) == MBUF_TC_BK);
2273 _CASSERT(MBUF_SC2TC(MBUF_SC_BE) == MBUF_TC_BE);
2274 _CASSERT(MBUF_SC2TC(MBUF_SC_RD) == MBUF_TC_BE);
2275 _CASSERT(MBUF_SC2TC(MBUF_SC_OAM) == MBUF_TC_BE);
2276 _CASSERT(MBUF_SC2TC(MBUF_SC_AV) == MBUF_TC_VI);
2277 _CASSERT(MBUF_SC2TC(MBUF_SC_RV) == MBUF_TC_VI);
2278 _CASSERT(MBUF_SC2TC(MBUF_SC_VI) == MBUF_TC_VI);
2279 _CASSERT(MBUF_SC2TC(MBUF_SC_SIG) == MBUF_TC_VI);
2280 _CASSERT(MBUF_SC2TC(MBUF_SC_VO) == MBUF_TC_VO);
2281 _CASSERT(MBUF_SC2TC(MBUF_SC_CTL) == MBUF_TC_VO);
2282
2283 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BK) == SCVAL_BK);
2284 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_BE) == SCVAL_BE);
2285 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VI) == SCVAL_VI);
2286 _CASSERT(MBUF_TC2SCVAL(MBUF_TC_VO) == SCVAL_VO);
2287
2288 /* Module specific scratch space (32-bit alignment requirement) */
2289 _CASSERT(!(offsetof(struct mbuf, m_pkthdr.pkt_mpriv) %
2290 sizeof(uint32_t)));
2291
2292 /* pktdata needs to start at 128-bit offset! */
2293 _CASSERT((offsetof(struct mbuf, m_pktdat) % 16) == 0);
2294
2295 /* Initialize random red zone cookie value */
2296 _CASSERT(sizeof(mb_redzone_cookie) ==
2297 sizeof(((struct pkthdr *)0)->redzone));
2298 read_random(&mb_redzone_cookie, sizeof(mb_redzone_cookie));
2299 read_random(&mb_obscure_extref, sizeof(mb_obscure_extref));
2300 read_random(&mb_obscure_extfree, sizeof(mb_obscure_extfree));
2301 mb_obscure_extref |= 0x3;
2302 mb_obscure_extref = 0;
2303 mb_obscure_extfree |= 0x3;
2304
2305 #if CONFIG_MBUF_MCACHE
2306 /* Make sure we don't save more than we should */
2307 _CASSERT(MCA_SAVED_MBUF_SIZE <= sizeof(struct mbuf));
2308 #endif /* CONFIG_MBUF_MCACHE */
2309
2310 if (nmbclusters == 0) {
2311 nmbclusters = NMBCLUSTERS;
2312 }
2313
2314 /* This should be a sane (at least even) value by now */
2315 VERIFY(nmbclusters != 0 && !(nmbclusters & 0x1));
2316
2317 /* Setup the mbuf table */
2318 mbuf_table_init();
2319
2320 _CASSERT(sizeof(struct mbuf) == _MSIZE);
2321
2322 #if CONFIG_MBUF_MCACHE
2323 /*
2324 * Allocate cluster slabs table:
2325 *
2326 * maxslabgrp = (N * 2048) / (1024 * 1024)
2327 *
2328 * Where N is nmbclusters rounded up to the nearest 512. This yields
2329 * mcl_slab_g_t units, each one representing a MB of memory.
2330 */
2331 maxslabgrp =
2332 (P2ROUNDUP(nmbclusters, (MBSIZE >> MCLSHIFT)) << MCLSHIFT) >> MBSHIFT;
2333 slabstbl = zalloc_permanent(maxslabgrp * sizeof(mcl_slabg_t *),
2334 ZALIGN(mcl_slabg_t));
2335
2336 /*
2337 * Allocate audit structures, if needed:
2338 *
2339 * maxclaudit = (maxslabgrp * 1024 * 1024) / PAGE_SIZE
2340 *
2341 * This yields mcl_audit_t units, each one representing a page.
2342 */
2343 PE_parse_boot_argn("mbuf_debug", &mbuf_debug, sizeof(mbuf_debug));
2344 mbuf_debug |= mcache_getflags();
2345 if (mbuf_debug & MCF_DEBUG) {
2346 int l;
2347 mcl_audit_t *mclad;
2348 maxclaudit = ((maxslabgrp << MBSHIFT) >> PAGE_SHIFT);
2349 mclaudit = zalloc_permanent(maxclaudit * sizeof(*mclaudit),
2350 ZALIGN(mcl_audit_t));
2351 for (l = 0, mclad = mclaudit; l < maxclaudit; l++) {
2352 mclad[l].cl_audit = zalloc_permanent(NMBPG * sizeof(mcache_audit_t *),
2353 ZALIGN_PTR);
2354 }
2355
2356 mcl_audit_con_cache = mcache_create("mcl_audit_contents",
2357 AUDIT_CONTENTS_SIZE, sizeof(u_int64_t), 0, MCR_SLEEP);
2358 VERIFY(mcl_audit_con_cache != NULL);
2359 }
2360 mclverify = (mbuf_debug & MCF_VERIFY);
2361 mcltrace = (mbuf_debug & MCF_TRACE);
2362 mclfindleak = !(mbuf_debug & MCF_NOLEAKLOG);
2363 mclexpleak = mclfindleak && (mbuf_debug & MCF_EXPLEAKLOG);
2364
2365 /* Enable mbuf leak logging, with a lock to protect the tables */
2366
2367 mleak_activate();
2368
2369 /*
2370 * Allocate structure for per-CPU statistics that's aligned
2371 * on the CPU cache boundary; this code assumes that we never
2372 * uninitialize this framework, since the original address
2373 * before alignment is not saved.
2374 */
2375 ncpu = ml_wait_max_cpus();
2376
2377 /* Calculate the number of pages assigned to the cluster pool */
2378 mcl_pages = (nmbclusters << MCLSHIFT) / PAGE_SIZE;
2379 mcl_paddr = zalloc_permanent(mcl_pages * sizeof(ppnum_t),
2380 ZALIGN(ppnum_t));
2381
2382 /* Register with the I/O Bus mapper */
2383 mcl_paddr_base = IOMapperIOVMAlloc(mcl_pages);
2384
2385 embutl = (mbutl + (nmbclusters * MCLBYTES));
2386 VERIFY(((embutl - mbutl) % MBIGCLBYTES) == 0);
2387
2388 /* Prime up the freelist */
2389 PE_parse_boot_argn("initmcl", &initmcl, sizeof(initmcl));
2390 if (initmcl != 0) {
2391 initmcl >>= NCLPBGSHIFT; /* become a 4K unit */
2392 if (initmcl > m_maxlimit(MC_BIGCL)) {
2393 initmcl = m_maxlimit(MC_BIGCL);
2394 }
2395 }
2396 if (initmcl < m_minlimit(MC_BIGCL)) {
2397 initmcl = m_minlimit(MC_BIGCL);
2398 }
2399
2400 lck_mtx_lock(mbuf_mlock);
2401
2402 /*
2403 * For classes with non-zero minimum limits, populate their freelists
2404 * so that m_total(class) is at least m_minlimit(class).
2405 */
2406 VERIFY(m_total(MC_BIGCL) == 0 && m_minlimit(MC_BIGCL) != 0);
2407 freelist_populate(m_class(MC_BIGCL), initmcl, M_WAIT);
2408 VERIFY(m_total(MC_BIGCL) >= m_minlimit(MC_BIGCL));
2409 freelist_init(m_class(MC_CL));
2410 #else
2411 /*
2412 * We have yet to create the non composite zones
2413 * and thus we haven't asked zalloc to allocate
2414 * anything yet, which means that at this point
2415 * m_total() is zero. Once we create the zones and
2416 * raise the reserve, m_total() will be calculated,
2417 * but until then just assume that we will have
2418 * at least the minium limit allocated.
2419 */
2420 m_total(MC_BIGCL) = m_minlimit(MC_BIGCL);
2421 m_total(MC_CL) = m_minlimit(MC_CL);
2422 #endif /* CONFIG_MBUF_MCACHE */
2423
2424 for (m = 0; m < NELEM(mbuf_table); m++) {
2425 /* Make sure we didn't miss any */
2426 VERIFY(m_minlimit(m_class(m)) == 0 ||
2427 m_total(m_class(m)) >= m_minlimit(m_class(m)));
2428 }
2429
2430 #if CONFIG_MBUF_MCACHE
2431 lck_mtx_unlock(mbuf_mlock);
2432
2433 (void) kernel_thread_start((thread_continue_t)mbuf_worker_thread_init,
2434 NULL, &thread);
2435 thread_deallocate(thread);
2436
2437 ref_cache = mcache_create("mext_ref", sizeof(struct ext_ref),
2438 0, 0, MCR_SLEEP);
2439 #endif /* CONFIG_MBUF_MCACHE */
2440
2441 /* Create the cache for each class */
2442 for (m = 0; m < NELEM(mbuf_table); m++) {
2443 #if CONFIG_MBUF_MCACHE
2444 void *allocfunc, *freefunc, *auditfunc, *logfunc;
2445 u_int32_t flags;
2446
2447 flags = mbuf_debug;
2448 if (m_class(m) == MC_MBUF_CL || m_class(m) == MC_MBUF_BIGCL ||
2449 m_class(m) == MC_MBUF_16KCL) {
2450 allocfunc = mbuf_cslab_alloc;
2451 freefunc = mbuf_cslab_free;
2452 auditfunc = mbuf_cslab_audit;
2453 logfunc = mleak_logger;
2454 } else {
2455 allocfunc = mbuf_slab_alloc;
2456 freefunc = mbuf_slab_free;
2457 auditfunc = mbuf_slab_audit;
2458 logfunc = mleak_logger;
2459 }
2460
2461 /*
2462 * Disable per-CPU caches for jumbo classes if there
2463 * is no jumbo cluster pool available in the system.
2464 * The cache itself is still created (but will never
2465 * be populated) since it simplifies the code.
2466 */
2467 if ((m_class(m) == MC_MBUF_16KCL || m_class(m) == MC_16KCL) &&
2468 njcl == 0) {
2469 flags |= MCF_NOCPUCACHE;
2470 }
2471
2472 if (!mclfindleak) {
2473 flags |= MCF_NOLEAKLOG;
2474 }
2475
2476 m_cache(m) = mcache_create_ext(m_cname(m), m_maxsize(m),
2477 allocfunc, freefunc, auditfunc, logfunc, mbuf_slab_notify,
2478 (void *)(uintptr_t)m, flags, MCR_SLEEP);
2479 #else
2480 if (!MBUF_CLASS_COMPOSITE(m)) {
2481 zone_t zone = zone_by_id(m_class_to_zid(m));
2482
2483 zone_set_exhaustible(zone, m_maxlimit(m), false);
2484 zone_raise_reserve(zone, m_minlimit(m));
2485 /*
2486 * Pretend that we have allocated m_total() items
2487 * at this point. zalloc will eventually do that
2488 * but it's an async operation.
2489 */
2490 m_total(m) = m_minlimit(m);
2491 }
2492 #endif /* CONFIG_MBUF_MCACHE */
2493 }
2494
2495 /*
2496 * Set the max limit on sb_max to be 1/16 th of the size of
2497 * memory allocated for mbuf clusters.
2498 */
2499 high_sb_max = (nmbclusters << (MCLSHIFT - 4));
2500 if (high_sb_max < sb_max) {
2501 /* sb_max is too large for this configuration, scale it down */
2502 if (high_sb_max > (1 << MBSHIFT)) {
2503 /* We have atleast 16 M of mbuf pool */
2504 sb_max = high_sb_max;
2505 } else if ((nmbclusters << MCLSHIFT) > (1 << MBSHIFT)) {
2506 /*
2507 * If we have more than 1M of mbufpool, cap the size of
2508 * max sock buf at 1M
2509 */
2510 sb_max = high_sb_max = (1 << MBSHIFT);
2511 } else {
2512 sb_max = high_sb_max;
2513 }
2514 }
2515
2516 #if CONFIG_MBUF_MCACHE
2517 /* allocate space for mbuf_dump_buf */
2518 mbuf_dump_buf = zalloc_permanent(MBUF_DUMP_BUF_SIZE, ZALIGN_NONE);
2519
2520 if (mbuf_debug & MCF_DEBUG) {
2521 printf("%s: MLEN %d, MHLEN %d\n", __func__,
2522 (int)_MLEN, (int)_MHLEN);
2523 }
2524 #else
2525 mbuf_defunct_tcall =
2526 thread_call_allocate_with_options(mbuf_watchdog_defunct,
2527 NULL,
2528 THREAD_CALL_PRIORITY_KERNEL,
2529 THREAD_CALL_OPTIONS_ONCE);
2530 mbuf_drain_tcall =
2531 thread_call_allocate_with_options(mbuf_watchdog_drain_composite,
2532 NULL,
2533 THREAD_CALL_PRIORITY_KERNEL,
2534 THREAD_CALL_OPTIONS_ONCE);
2535 #endif /* CONFIG_MBUF_MCACHE */
2536 printf("%s: done [%d MB total pool size, (%d/%d) split]\n", __func__,
2537 (nmbclusters << MCLSHIFT) >> MBSHIFT,
2538 (nclusters << MCLSHIFT) >> MBSHIFT,
2539 (njcl << MCLSHIFT) >> MBSHIFT);
2540 }
2541
2542 #if CONFIG_MBUF_MCACHE
2543 /*
2544 * Obtain a slab of object(s) from the class's freelist.
2545 */
2546 static mcache_obj_t *
slab_alloc(mbuf_class_t class,int wait)2547 slab_alloc(mbuf_class_t class, int wait)
2548 {
2549 mcl_slab_t *sp;
2550 mcache_obj_t *buf;
2551
2552 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2553
2554 /* This should always be NULL for us */
2555 VERIFY(m_cobjlist(class) == NULL);
2556
2557 /*
2558 * Treat composite objects as having longer lifespan by using
2559 * a slab from the reverse direction, in hoping that this could
2560 * reduce the probability of fragmentation for slabs that hold
2561 * more than one buffer chunks (e.g. mbuf slabs). For other
2562 * slabs, this probably doesn't make much of a difference.
2563 */
2564 if ((class == MC_MBUF || class == MC_CL || class == MC_BIGCL)
2565 && (wait & MCR_COMP)) {
2566 sp = (mcl_slab_t *)TAILQ_LAST(&m_slablist(class), mcl_slhead);
2567 } else {
2568 sp = (mcl_slab_t *)TAILQ_FIRST(&m_slablist(class));
2569 }
2570
2571 if (sp == NULL) {
2572 VERIFY(m_infree(class) == 0 && m_slab_cnt(class) == 0);
2573 /* The slab list for this class is empty */
2574 return NULL;
2575 }
2576
2577 VERIFY(m_infree(class) > 0);
2578 VERIFY(!slab_is_detached(sp));
2579 VERIFY(sp->sl_class == class &&
2580 (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
2581 buf = sp->sl_head;
2582 VERIFY(slab_inrange(sp, buf) && sp == slab_get(buf));
2583 sp->sl_head = buf->obj_next;
2584 /* Increment slab reference */
2585 sp->sl_refcnt++;
2586
2587 VERIFY(sp->sl_head != NULL || sp->sl_refcnt == sp->sl_chunks);
2588
2589 if (sp->sl_head != NULL && !slab_inrange(sp, sp->sl_head)) {
2590 slab_nextptr_panic(sp, sp->sl_head);
2591 /* In case sl_head is in the map but not in the slab */
2592 VERIFY(slab_inrange(sp, sp->sl_head));
2593 /* NOTREACHED */
2594 }
2595
2596 if (mclaudit != NULL) {
2597 mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
2598 mca->mca_uflags = 0;
2599 /* Save contents on mbuf objects only */
2600 if (class == MC_MBUF) {
2601 mca->mca_uflags |= MB_SCVALID;
2602 }
2603 }
2604
2605 if (class == MC_CL) {
2606 mbstat.m_clfree = (--m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
2607 /*
2608 * A 2K cluster slab can have at most NCLPG references.
2609 */
2610 VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NCLPG &&
2611 sp->sl_chunks == NCLPG && sp->sl_len == PAGE_SIZE);
2612 VERIFY(sp->sl_refcnt < NCLPG || sp->sl_head == NULL);
2613 } else if (class == MC_BIGCL) {
2614 mbstat.m_bigclfree = (--m_infree(MC_BIGCL)) +
2615 m_infree(MC_MBUF_BIGCL);
2616 /*
2617 * A 4K cluster slab can have NBCLPG references.
2618 */
2619 VERIFY(sp->sl_refcnt >= 1 && sp->sl_chunks == NBCLPG &&
2620 sp->sl_len == PAGE_SIZE &&
2621 (sp->sl_refcnt < NBCLPG || sp->sl_head == NULL));
2622 } else if (class == MC_16KCL) {
2623 mcl_slab_t *nsp;
2624 int k;
2625
2626 --m_infree(MC_16KCL);
2627 VERIFY(sp->sl_refcnt == 1 && sp->sl_chunks == 1 &&
2628 sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
2629 /*
2630 * Increment 2nd-Nth slab reference, where N is NSLABSP16KB.
2631 * A 16KB big cluster takes NSLABSP16KB slabs, each having at
2632 * most 1 reference.
2633 */
2634 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
2635 nsp = nsp->sl_next;
2636 /* Next slab must already be present */
2637 VERIFY(nsp != NULL);
2638 nsp->sl_refcnt++;
2639 VERIFY(!slab_is_detached(nsp));
2640 VERIFY(nsp->sl_class == MC_16KCL &&
2641 nsp->sl_flags == (SLF_MAPPED | SLF_PARTIAL) &&
2642 nsp->sl_refcnt == 1 && nsp->sl_chunks == 0 &&
2643 nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
2644 nsp->sl_head == NULL);
2645 }
2646 } else {
2647 VERIFY(class == MC_MBUF);
2648 --m_infree(MC_MBUF);
2649 /*
2650 * If auditing is turned on, this check is
2651 * deferred until later in mbuf_slab_audit().
2652 */
2653 if (mclaudit == NULL) {
2654 _MCHECK((struct mbuf *)buf);
2655 }
2656 /*
2657 * Since we have incremented the reference count above,
2658 * an mbuf slab (formerly a 4KB cluster slab that was cut
2659 * up into mbufs) must have a reference count between 1
2660 * and NMBPG at this point.
2661 */
2662 VERIFY(sp->sl_refcnt >= 1 && sp->sl_refcnt <= NMBPG &&
2663 sp->sl_chunks == NMBPG &&
2664 sp->sl_len == PAGE_SIZE);
2665 VERIFY(sp->sl_refcnt < NMBPG || sp->sl_head == NULL);
2666 }
2667
2668 /* If empty, remove this slab from the class's freelist */
2669 if (sp->sl_head == NULL) {
2670 VERIFY(class != MC_MBUF || sp->sl_refcnt == NMBPG);
2671 VERIFY(class != MC_CL || sp->sl_refcnt == NCLPG);
2672 VERIFY(class != MC_BIGCL || sp->sl_refcnt == NBCLPG);
2673 slab_remove(sp, class);
2674 }
2675
2676 return buf;
2677 }
2678
2679 /*
2680 * Place a slab of object(s) back into a class's slab list.
2681 */
2682 static void
slab_free(mbuf_class_t class,mcache_obj_t * buf)2683 slab_free(mbuf_class_t class, mcache_obj_t *buf)
2684 {
2685 mcl_slab_t *sp;
2686 boolean_t reinit_supercl = false;
2687 mbuf_class_t super_class;
2688
2689 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2690
2691 VERIFY(class != MC_16KCL || njcl > 0);
2692 VERIFY(buf->obj_next == NULL);
2693
2694 /*
2695 * Synchronizing with m_clalloc, as it reads m_total, while we here
2696 * are modifying m_total.
2697 */
2698 while (mb_clalloc_busy) {
2699 mb_clalloc_waiters++;
2700 (void) msleep(mb_clalloc_waitchan, mbuf_mlock,
2701 (PZERO - 1), "m_clalloc", NULL);
2702 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2703 }
2704
2705 /* We are busy now; tell everyone else to go away */
2706 mb_clalloc_busy = TRUE;
2707
2708 sp = slab_get(buf);
2709 VERIFY(sp->sl_class == class && slab_inrange(sp, buf) &&
2710 (sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
2711
2712 /* Decrement slab reference */
2713 sp->sl_refcnt--;
2714
2715 if (class == MC_CL) {
2716 VERIFY(IS_P2ALIGNED(buf, MCLBYTES));
2717 /*
2718 * A slab that has been splitted for 2KB clusters can have
2719 * at most 1 outstanding reference at this point.
2720 */
2721 VERIFY(sp->sl_refcnt >= 0 && sp->sl_refcnt <= (NCLPG - 1) &&
2722 sp->sl_chunks == NCLPG && sp->sl_len == PAGE_SIZE);
2723 VERIFY(sp->sl_refcnt < (NCLPG - 1) ||
2724 (slab_is_detached(sp) && sp->sl_head == NULL));
2725 } else if (class == MC_BIGCL) {
2726 VERIFY(IS_P2ALIGNED(buf, MBIGCLBYTES));
2727
2728 /* A 4KB cluster slab can have NBCLPG references at most */
2729 VERIFY(sp->sl_refcnt >= 0 && sp->sl_chunks == NBCLPG);
2730 VERIFY(sp->sl_refcnt < (NBCLPG - 1) ||
2731 (slab_is_detached(sp) && sp->sl_head == NULL));
2732 } else if (class == MC_16KCL) {
2733 mcl_slab_t *nsp;
2734 int k;
2735 /*
2736 * A 16KB cluster takes NSLABSP16KB slabs, all must
2737 * now have 0 reference.
2738 */
2739 VERIFY(IS_P2ALIGNED(buf, PAGE_SIZE));
2740 VERIFY(sp->sl_refcnt == 0 && sp->sl_chunks == 1 &&
2741 sp->sl_len == m_maxsize(class) && sp->sl_head == NULL);
2742 VERIFY(slab_is_detached(sp));
2743 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
2744 nsp = nsp->sl_next;
2745 /* Next slab must already be present */
2746 VERIFY(nsp != NULL);
2747 nsp->sl_refcnt--;
2748 VERIFY(slab_is_detached(nsp));
2749 VERIFY(nsp->sl_class == MC_16KCL &&
2750 (nsp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) &&
2751 nsp->sl_refcnt == 0 && nsp->sl_chunks == 0 &&
2752 nsp->sl_len == 0 && nsp->sl_base == sp->sl_base &&
2753 nsp->sl_head == NULL);
2754 }
2755 } else {
2756 /*
2757 * A slab that has been splitted for mbufs has at most
2758 * NMBPG reference counts. Since we have decremented
2759 * one reference above, it must now be between 0 and
2760 * NMBPG-1.
2761 */
2762 VERIFY(class == MC_MBUF);
2763 VERIFY(sp->sl_refcnt >= 0 &&
2764 sp->sl_refcnt <= (NMBPG - 1) &&
2765 sp->sl_chunks == NMBPG &&
2766 sp->sl_len == PAGE_SIZE);
2767 VERIFY(sp->sl_refcnt < (NMBPG - 1) ||
2768 (slab_is_detached(sp) && sp->sl_head == NULL));
2769 }
2770
2771 /*
2772 * When auditing is enabled, ensure that the buffer still
2773 * contains the free pattern. Otherwise it got corrupted
2774 * while at the CPU cache layer.
2775 */
2776 if (mclaudit != NULL) {
2777 mcache_audit_t *mca = mcl_audit_buf2mca(class, buf);
2778 if (mclverify) {
2779 mcache_audit_free_verify(mca, buf, 0,
2780 m_maxsize(class));
2781 }
2782 mca->mca_uflags &= ~MB_SCVALID;
2783 }
2784
2785 if (class == MC_CL) {
2786 mbstat.m_clfree = (++m_infree(MC_CL)) + m_infree(MC_MBUF_CL);
2787 buf->obj_next = sp->sl_head;
2788 } else if (class == MC_BIGCL) {
2789 mbstat.m_bigclfree = (++m_infree(MC_BIGCL)) +
2790 m_infree(MC_MBUF_BIGCL);
2791 buf->obj_next = sp->sl_head;
2792 } else if (class == MC_16KCL) {
2793 ++m_infree(MC_16KCL);
2794 } else {
2795 ++m_infree(MC_MBUF);
2796 buf->obj_next = sp->sl_head;
2797 }
2798 sp->sl_head = buf;
2799
2800 /*
2801 * If a slab has been split to either one which holds 2KB clusters,
2802 * or one which holds mbufs, turn it back to one which holds a
2803 * 4 or 16 KB cluster depending on the page size.
2804 */
2805 if (m_maxsize(MC_BIGCL) == PAGE_SIZE) {
2806 super_class = MC_BIGCL;
2807 } else {
2808 VERIFY(PAGE_SIZE == m_maxsize(MC_16KCL));
2809 super_class = MC_16KCL;
2810 }
2811 if (class == MC_MBUF && sp->sl_refcnt == 0 &&
2812 m_total(class) >= (m_minlimit(class) + NMBPG) &&
2813 m_total(super_class) < m_maxlimit(super_class)) {
2814 int i = NMBPG;
2815
2816 m_total(MC_MBUF) -= NMBPG;
2817 mbstat.m_mbufs = m_total(MC_MBUF);
2818 m_infree(MC_MBUF) -= NMBPG;
2819 mtype_stat_add(MT_FREE, -((unsigned)NMBPG));
2820
2821 while (i--) {
2822 struct mbuf *m = sp->sl_head;
2823 VERIFY(m != NULL);
2824 sp->sl_head = m->m_next;
2825 m->m_next = NULL;
2826 }
2827 reinit_supercl = true;
2828 } else if (class == MC_CL && sp->sl_refcnt == 0 &&
2829 m_total(class) >= (m_minlimit(class) + NCLPG) &&
2830 m_total(super_class) < m_maxlimit(super_class)) {
2831 int i = NCLPG;
2832
2833 m_total(MC_CL) -= NCLPG;
2834 mbstat.m_clusters = m_total(MC_CL);
2835 m_infree(MC_CL) -= NCLPG;
2836
2837 while (i--) {
2838 union mcluster *c = sp->sl_head;
2839 VERIFY(c != NULL);
2840 sp->sl_head = c->mcl_next;
2841 c->mcl_next = NULL;
2842 }
2843 reinit_supercl = true;
2844 } else if (class == MC_BIGCL && super_class != MC_BIGCL &&
2845 sp->sl_refcnt == 0 &&
2846 m_total(class) >= (m_minlimit(class) + NBCLPG) &&
2847 m_total(super_class) < m_maxlimit(super_class)) {
2848 int i = NBCLPG;
2849
2850 VERIFY(super_class == MC_16KCL);
2851 m_total(MC_BIGCL) -= NBCLPG;
2852 mbstat.m_bigclusters = m_total(MC_BIGCL);
2853 m_infree(MC_BIGCL) -= NBCLPG;
2854
2855 while (i--) {
2856 union mbigcluster *bc = sp->sl_head;
2857 VERIFY(bc != NULL);
2858 sp->sl_head = bc->mbc_next;
2859 bc->mbc_next = NULL;
2860 }
2861 reinit_supercl = true;
2862 }
2863
2864 if (reinit_supercl) {
2865 VERIFY(sp->sl_head == NULL);
2866 VERIFY(m_total(class) >= m_minlimit(class));
2867 slab_remove(sp, class);
2868
2869 /* Reinitialize it as a cluster for the super class */
2870 m_total(super_class)++;
2871 m_infree(super_class)++;
2872 VERIFY(sp->sl_flags == (SLF_MAPPED | SLF_DETACHED) &&
2873 sp->sl_len == PAGE_SIZE && sp->sl_refcnt == 0);
2874
2875 slab_init(sp, super_class, SLF_MAPPED, sp->sl_base,
2876 sp->sl_base, PAGE_SIZE, 0, 1);
2877 if (mclverify) {
2878 mcache_set_pattern(MCACHE_FREE_PATTERN,
2879 (caddr_t)sp->sl_base, sp->sl_len);
2880 }
2881 ((mcache_obj_t *)(sp->sl_base))->obj_next = NULL;
2882
2883 if (super_class == MC_BIGCL) {
2884 mbstat.m_bigclusters = m_total(MC_BIGCL);
2885 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
2886 m_infree(MC_MBUF_BIGCL);
2887 }
2888
2889 VERIFY(slab_is_detached(sp));
2890 VERIFY(m_total(super_class) <= m_maxlimit(super_class));
2891
2892 /* And finally switch class */
2893 class = super_class;
2894 }
2895
2896 /* Reinsert the slab to the class's slab list */
2897 if (slab_is_detached(sp)) {
2898 slab_insert(sp, class);
2899 }
2900
2901 /* We're done; let others enter */
2902 mb_clalloc_busy = FALSE;
2903 if (mb_clalloc_waiters > 0) {
2904 mb_clalloc_waiters = 0;
2905 wakeup(mb_clalloc_waitchan);
2906 }
2907 }
2908
2909 /*
2910 * Common allocator for rudimentary objects called by the CPU cache layer
2911 * during an allocation request whenever there is no available element in the
2912 * bucket layer. It returns one or more elements from the appropriate global
2913 * freelist. If the freelist is empty, it will attempt to populate it and
2914 * retry the allocation.
2915 */
2916 static unsigned int
mbuf_slab_alloc(void * arg,mcache_obj_t *** plist,unsigned int num,int wait)2917 mbuf_slab_alloc(void *arg, mcache_obj_t ***plist, unsigned int num, int wait)
2918 {
2919 mbuf_class_t class = (mbuf_class_t)arg;
2920 unsigned int need = num;
2921 mcache_obj_t **list = *plist;
2922
2923 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
2924 ASSERT(need > 0);
2925
2926 lck_mtx_lock(mbuf_mlock);
2927
2928 for (;;) {
2929 if ((*list = slab_alloc(class, wait)) != NULL) {
2930 (*list)->obj_next = NULL;
2931 list = *plist = &(*list)->obj_next;
2932
2933 if (--need == 0) {
2934 /*
2935 * If the number of elements in freelist has
2936 * dropped below low watermark, asynchronously
2937 * populate the freelist now rather than doing
2938 * it later when we run out of elements.
2939 */
2940 if (!mbuf_cached_above(class, wait) &&
2941 m_infree(class) < (m_total(class) >> 5)) {
2942 (void) freelist_populate(class, 1,
2943 M_DONTWAIT);
2944 }
2945 break;
2946 }
2947 } else {
2948 VERIFY(m_infree(class) == 0 || class == MC_CL);
2949
2950 (void) freelist_populate(class, 1,
2951 (wait & MCR_NOSLEEP) ? M_DONTWAIT : M_WAIT);
2952
2953 if (m_infree(class) > 0) {
2954 continue;
2955 }
2956
2957 /* Check if there's anything at the cache layer */
2958 if (mbuf_cached_above(class, wait)) {
2959 break;
2960 }
2961
2962 /* watchdog checkpoint */
2963 mbuf_watchdog();
2964
2965 /* We have nothing and cannot block; give up */
2966 if (wait & MCR_NOSLEEP) {
2967 if (!(wait & MCR_TRYHARD)) {
2968 m_fail_cnt(class)++;
2969 mbstat.m_drops++;
2970 break;
2971 }
2972 }
2973
2974 /*
2975 * If the freelist is still empty and the caller is
2976 * willing to be blocked, sleep on the wait channel
2977 * until an element is available. Otherwise, if
2978 * MCR_TRYHARD is set, do our best to satisfy the
2979 * request without having to go to sleep.
2980 */
2981 if (mbuf_worker_ready &&
2982 mbuf_sleep(class, need, wait)) {
2983 break;
2984 }
2985
2986 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
2987 }
2988 }
2989
2990 m_alloc_cnt(class) += num - need;
2991 lck_mtx_unlock(mbuf_mlock);
2992
2993 return num - need;
2994 }
2995
2996 /*
2997 * Common de-allocator for rudimentary objects called by the CPU cache
2998 * layer when one or more elements need to be returned to the appropriate
2999 * global freelist.
3000 */
3001 static void
mbuf_slab_free(void * arg,mcache_obj_t * list,__unused int purged)3002 mbuf_slab_free(void *arg, mcache_obj_t *list, __unused int purged)
3003 {
3004 mbuf_class_t class = (mbuf_class_t)arg;
3005 mcache_obj_t *nlist;
3006 unsigned int num = 0;
3007 int w;
3008
3009 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
3010
3011 lck_mtx_lock(mbuf_mlock);
3012
3013 for (;;) {
3014 nlist = list->obj_next;
3015 list->obj_next = NULL;
3016 slab_free(class, list);
3017 ++num;
3018 if ((list = nlist) == NULL) {
3019 break;
3020 }
3021 }
3022 m_free_cnt(class) += num;
3023
3024 if ((w = mb_waiters) > 0) {
3025 mb_waiters = 0;
3026 }
3027 if (w) {
3028 mbwdog_logger("waking up all threads");
3029 }
3030 lck_mtx_unlock(mbuf_mlock);
3031
3032 if (w != 0) {
3033 wakeup(mb_waitchan);
3034 }
3035 }
3036
3037 /*
3038 * Common auditor for rudimentary objects called by the CPU cache layer
3039 * during an allocation or free request. For the former, this is called
3040 * after the objects are obtained from either the bucket or slab layer
3041 * and before they are returned to the caller. For the latter, this is
3042 * called immediately during free and before placing the objects into
3043 * the bucket or slab layer.
3044 */
3045 static void
mbuf_slab_audit(void * arg,mcache_obj_t * list,boolean_t alloc)3046 mbuf_slab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
3047 {
3048 mbuf_class_t class = (mbuf_class_t)arg;
3049 mcache_audit_t *mca;
3050
3051 ASSERT(MBUF_CLASS_VALID(class) && !MBUF_CLASS_COMPOSITE(class));
3052
3053 while (list != NULL) {
3054 lck_mtx_lock(mbuf_mlock);
3055 mca = mcl_audit_buf2mca(class, list);
3056
3057 /* Do the sanity checks */
3058 if (class == MC_MBUF) {
3059 mcl_audit_mbuf(mca, list, FALSE, alloc);
3060 ASSERT(mca->mca_uflags & MB_SCVALID);
3061 } else {
3062 mcl_audit_cluster(mca, list, m_maxsize(class),
3063 alloc, TRUE);
3064 ASSERT(!(mca->mca_uflags & MB_SCVALID));
3065 }
3066 /* Record this transaction */
3067 if (mcltrace) {
3068 mcache_buffer_log(mca, list, m_cache(class), &mb_start);
3069 }
3070
3071 if (alloc) {
3072 mca->mca_uflags |= MB_INUSE;
3073 } else {
3074 mca->mca_uflags &= ~MB_INUSE;
3075 }
3076 /* Unpair the object (unconditionally) */
3077 mca->mca_uptr = NULL;
3078 lck_mtx_unlock(mbuf_mlock);
3079
3080 list = list->obj_next;
3081 }
3082 }
3083
3084 /*
3085 * Common notify routine for all caches. It is called by mcache when
3086 * one or more objects get freed. We use this indication to trigger
3087 * the wakeup of any sleeping threads so that they can retry their
3088 * allocation requests.
3089 */
3090 static void
mbuf_slab_notify(void * arg,u_int32_t reason)3091 mbuf_slab_notify(void *arg, u_int32_t reason)
3092 {
3093 mbuf_class_t class = (mbuf_class_t)arg;
3094 int w;
3095
3096 ASSERT(MBUF_CLASS_VALID(class));
3097
3098 if (reason != MCN_RETRYALLOC) {
3099 return;
3100 }
3101
3102 lck_mtx_lock(mbuf_mlock);
3103 if ((w = mb_waiters) > 0) {
3104 m_notified(class)++;
3105 mb_waiters = 0;
3106 }
3107 if (w) {
3108 mbwdog_logger("waking up all threads");
3109 }
3110 lck_mtx_unlock(mbuf_mlock);
3111
3112 if (w != 0) {
3113 wakeup(mb_waitchan);
3114 }
3115 }
3116
3117 /*
3118 * Obtain object(s) from the composite class's freelist.
3119 */
3120 static unsigned int
cslab_alloc(mbuf_class_t class,mcache_obj_t *** plist,unsigned int num)3121 cslab_alloc(mbuf_class_t class, mcache_obj_t ***plist, unsigned int num)
3122 {
3123 unsigned int need = num;
3124 mcl_slab_t *sp, *clsp, *nsp;
3125 struct mbuf *m;
3126 mcache_obj_t **list = *plist;
3127 void *cl;
3128
3129 VERIFY(need > 0);
3130 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
3131 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3132
3133 /* Get what we can from the freelist */
3134 while ((*list = m_cobjlist(class)) != NULL) {
3135 MRANGE(*list);
3136
3137 m = (struct mbuf *)*list;
3138 sp = slab_get(m);
3139 cl = m->m_ext.ext_buf;
3140 clsp = slab_get(cl);
3141 VERIFY(m->m_flags == M_EXT && cl != NULL);
3142 VERIFY(m_get_rfa(m) != NULL && MBUF_IS_COMPOSITE(m));
3143
3144 if (class == MC_MBUF_CL) {
3145 VERIFY(clsp->sl_refcnt >= 1 &&
3146 clsp->sl_refcnt <= NCLPG);
3147 } else {
3148 VERIFY(clsp->sl_refcnt >= 1 &&
3149 clsp->sl_refcnt <= NBCLPG);
3150 }
3151
3152 if (class == MC_MBUF_16KCL) {
3153 int k;
3154 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
3155 nsp = nsp->sl_next;
3156 /* Next slab must already be present */
3157 VERIFY(nsp != NULL);
3158 VERIFY(nsp->sl_refcnt == 1);
3159 }
3160 }
3161
3162 if ((m_cobjlist(class) = (*list)->obj_next) != NULL &&
3163 !MBUF_IN_MAP(m_cobjlist(class))) {
3164 slab_nextptr_panic(sp, m_cobjlist(class));
3165 /* NOTREACHED */
3166 }
3167 (*list)->obj_next = NULL;
3168 list = *plist = &(*list)->obj_next;
3169
3170 if (--need == 0) {
3171 break;
3172 }
3173 }
3174 m_infree(class) -= (num - need);
3175
3176 return num - need;
3177 }
3178
3179 /*
3180 * Place object(s) back into a composite class's freelist.
3181 */
3182 static unsigned int
cslab_free(mbuf_class_t class,mcache_obj_t * list,int purged)3183 cslab_free(mbuf_class_t class, mcache_obj_t *list, int purged)
3184 {
3185 mcache_obj_t *o, *tail;
3186 unsigned int num = 0;
3187 struct mbuf *m, *ms;
3188 mcache_audit_t *mca = NULL;
3189 mcache_obj_t *ref_list = NULL;
3190 mcl_slab_t *clsp, *nsp;
3191 void *cl;
3192 mbuf_class_t cl_class;
3193
3194 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
3195 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
3196 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3197
3198 if (class == MC_MBUF_CL) {
3199 cl_class = MC_CL;
3200 } else if (class == MC_MBUF_BIGCL) {
3201 cl_class = MC_BIGCL;
3202 } else {
3203 VERIFY(class == MC_MBUF_16KCL);
3204 cl_class = MC_16KCL;
3205 }
3206
3207 o = tail = list;
3208
3209 while ((m = ms = (struct mbuf *)o) != NULL) {
3210 mcache_obj_t *rfa, *nexto = o->obj_next;
3211
3212 /* Do the mbuf sanity checks */
3213 if (mclaudit != NULL) {
3214 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
3215 if (mclverify) {
3216 mcache_audit_free_verify(mca, m, 0,
3217 m_maxsize(MC_MBUF));
3218 }
3219 ms = MCA_SAVED_MBUF_PTR(mca);
3220 }
3221
3222 /* Do the cluster sanity checks */
3223 cl = ms->m_ext.ext_buf;
3224 clsp = slab_get(cl);
3225 if (mclverify) {
3226 size_t size = m_maxsize(cl_class);
3227 mcache_audit_free_verify(mcl_audit_buf2mca(cl_class,
3228 (mcache_obj_t *)cl), cl, 0, size);
3229 }
3230 VERIFY(ms->m_type == MT_FREE);
3231 VERIFY(ms->m_flags == M_EXT);
3232 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
3233 if (cl_class == MC_CL) {
3234 VERIFY(clsp->sl_refcnt >= 1 &&
3235 clsp->sl_refcnt <= NCLPG);
3236 } else {
3237 VERIFY(clsp->sl_refcnt >= 1 &&
3238 clsp->sl_refcnt <= NBCLPG);
3239 }
3240 if (cl_class == MC_16KCL) {
3241 int k;
3242 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
3243 nsp = nsp->sl_next;
3244 /* Next slab must already be present */
3245 VERIFY(nsp != NULL);
3246 VERIFY(nsp->sl_refcnt == 1);
3247 }
3248 }
3249
3250 /*
3251 * If we're asked to purge, restore the actual mbuf using
3252 * contents of the shadow structure (if auditing is enabled)
3253 * and clear EXTF_COMPOSITE flag from the mbuf, as we are
3254 * about to free it and the attached cluster into their caches.
3255 */
3256 if (purged) {
3257 /* Restore constructed mbuf fields */
3258 if (mclaudit != NULL) {
3259 mcl_audit_restore_mbuf(m, mca, TRUE);
3260 }
3261
3262 MEXT_MINREF(m) = 0;
3263 MEXT_REF(m) = 0;
3264 MEXT_PREF(m) = 0;
3265 MEXT_FLAGS(m) = 0;
3266 MEXT_PRIV(m) = 0;
3267 MEXT_PMBUF(m) = NULL;
3268 MEXT_TOKEN(m) = 0;
3269
3270 rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
3271 m_set_ext(m, NULL, NULL, NULL);
3272 rfa->obj_next = ref_list;
3273 ref_list = rfa;
3274
3275 m->m_type = MT_FREE;
3276 m->m_flags = m->m_len = 0;
3277 m->m_next = m->m_nextpkt = NULL;
3278
3279 /* Save mbuf fields and make auditing happy */
3280 if (mclaudit != NULL) {
3281 mcl_audit_mbuf(mca, o, FALSE, FALSE);
3282 }
3283
3284 VERIFY(m_total(class) > 0);
3285 m_total(class)--;
3286
3287 /* Free the mbuf */
3288 o->obj_next = NULL;
3289 slab_free(MC_MBUF, o);
3290
3291 /* And free the cluster */
3292 ((mcache_obj_t *)cl)->obj_next = NULL;
3293 if (class == MC_MBUF_CL) {
3294 slab_free(MC_CL, cl);
3295 } else if (class == MC_MBUF_BIGCL) {
3296 slab_free(MC_BIGCL, cl);
3297 } else {
3298 slab_free(MC_16KCL, cl);
3299 }
3300 }
3301
3302 ++num;
3303 tail = o;
3304 o = nexto;
3305 }
3306
3307 if (!purged) {
3308 tail->obj_next = m_cobjlist(class);
3309 m_cobjlist(class) = list;
3310 m_infree(class) += num;
3311 } else if (ref_list != NULL) {
3312 mcache_free_ext(ref_cache, ref_list);
3313 }
3314
3315 return num;
3316 }
3317
3318 /*
3319 * Common allocator for composite objects called by the CPU cache layer
3320 * during an allocation request whenever there is no available element in
3321 * the bucket layer. It returns one or more composite elements from the
3322 * appropriate global freelist. If the freelist is empty, it will attempt
3323 * to obtain the rudimentary objects from their caches and construct them
3324 * into composite mbuf + cluster objects.
3325 */
3326 static unsigned int
mbuf_cslab_alloc(void * arg,mcache_obj_t *** plist,unsigned int needed,int wait)3327 mbuf_cslab_alloc(void *arg, mcache_obj_t ***plist, unsigned int needed,
3328 int wait)
3329 {
3330 mbuf_class_t class = (mbuf_class_t)arg;
3331 mbuf_class_t cl_class = 0;
3332 unsigned int num = 0, cnum = 0, want = needed;
3333 mcache_obj_t *ref_list = NULL;
3334 mcache_obj_t *mp_list = NULL;
3335 mcache_obj_t *clp_list = NULL;
3336 mcache_obj_t **list;
3337 struct ext_ref *rfa;
3338 struct mbuf *m;
3339 void *cl;
3340
3341 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
3342 ASSERT(needed > 0);
3343
3344 VERIFY(class != MC_MBUF_16KCL || njcl > 0);
3345
3346 /* There should not be any slab for this class */
3347 VERIFY(m_slab_cnt(class) == 0 &&
3348 m_slablist(class).tqh_first == NULL &&
3349 m_slablist(class).tqh_last == NULL);
3350
3351 lck_mtx_lock(mbuf_mlock);
3352
3353 /* Try using the freelist first */
3354 num = cslab_alloc(class, plist, needed);
3355 list = *plist;
3356 if (num == needed) {
3357 m_alloc_cnt(class) += num;
3358 lck_mtx_unlock(mbuf_mlock);
3359 return needed;
3360 }
3361
3362 lck_mtx_unlock(mbuf_mlock);
3363
3364 /*
3365 * We could not satisfy the request using the freelist alone;
3366 * allocate from the appropriate rudimentary caches and use
3367 * whatever we can get to construct the composite objects.
3368 */
3369 needed -= num;
3370
3371 /*
3372 * Mark these allocation requests as coming from a composite cache.
3373 * Also, if the caller is willing to be blocked, mark the request
3374 * with MCR_FAILOK such that we don't end up sleeping at the mbuf
3375 * slab layer waiting for the individual object when one or more
3376 * of the already-constructed composite objects are available.
3377 */
3378 wait |= MCR_COMP;
3379 if (!(wait & MCR_NOSLEEP)) {
3380 wait |= MCR_FAILOK;
3381 }
3382
3383 /* allocate mbufs */
3384 needed = mcache_alloc_ext(m_cache(MC_MBUF), &mp_list, needed, wait);
3385 if (needed == 0) {
3386 ASSERT(mp_list == NULL);
3387 goto fail;
3388 }
3389
3390 /* allocate clusters */
3391 if (class == MC_MBUF_CL) {
3392 cl_class = MC_CL;
3393 } else if (class == MC_MBUF_BIGCL) {
3394 cl_class = MC_BIGCL;
3395 } else {
3396 VERIFY(class == MC_MBUF_16KCL);
3397 cl_class = MC_16KCL;
3398 }
3399 needed = mcache_alloc_ext(m_cache(cl_class), &clp_list, needed, wait);
3400 if (needed == 0) {
3401 ASSERT(clp_list == NULL);
3402 goto fail;
3403 }
3404
3405 needed = mcache_alloc_ext(ref_cache, &ref_list, needed, wait);
3406 if (needed == 0) {
3407 ASSERT(ref_list == NULL);
3408 goto fail;
3409 }
3410
3411 /*
3412 * By this time "needed" is MIN(mbuf, cluster, ref). Any left
3413 * overs will get freed accordingly before we return to caller.
3414 */
3415 for (cnum = 0; cnum < needed; cnum++) {
3416 struct mbuf *ms;
3417
3418 m = ms = (struct mbuf *)mp_list;
3419 mp_list = mp_list->obj_next;
3420
3421 cl = clp_list;
3422 clp_list = clp_list->obj_next;
3423 ((mcache_obj_t *)cl)->obj_next = NULL;
3424
3425 rfa = (struct ext_ref *)ref_list;
3426 ref_list = ref_list->obj_next;
3427 ((mcache_obj_t *)(void *)rfa)->obj_next = NULL;
3428
3429 /*
3430 * If auditing is enabled, construct the shadow mbuf
3431 * in the audit structure instead of in the actual one.
3432 * mbuf_cslab_audit() will take care of restoring the
3433 * contents after the integrity check.
3434 */
3435 if (mclaudit != NULL) {
3436 mcache_audit_t *mca, *cl_mca;
3437
3438 lck_mtx_lock(mbuf_mlock);
3439 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
3440 ms = MCA_SAVED_MBUF_PTR(mca);
3441 cl_mca = mcl_audit_buf2mca(cl_class,
3442 (mcache_obj_t *)cl);
3443
3444 /*
3445 * Pair them up. Note that this is done at the time
3446 * the mbuf+cluster objects are constructed. This
3447 * information should be treated as "best effort"
3448 * debugging hint since more than one mbufs can refer
3449 * to a cluster. In that case, the cluster might not
3450 * be freed along with the mbuf it was paired with.
3451 */
3452 mca->mca_uptr = cl_mca;
3453 cl_mca->mca_uptr = mca;
3454
3455 ASSERT(mca->mca_uflags & MB_SCVALID);
3456 ASSERT(!(cl_mca->mca_uflags & MB_SCVALID));
3457 lck_mtx_unlock(mbuf_mlock);
3458
3459 /* Technically, they are in the freelist */
3460 if (mclverify) {
3461 size_t size;
3462
3463 mcache_set_pattern(MCACHE_FREE_PATTERN, m,
3464 m_maxsize(MC_MBUF));
3465
3466 if (class == MC_MBUF_CL) {
3467 size = m_maxsize(MC_CL);
3468 } else if (class == MC_MBUF_BIGCL) {
3469 size = m_maxsize(MC_BIGCL);
3470 } else {
3471 size = m_maxsize(MC_16KCL);
3472 }
3473
3474 mcache_set_pattern(MCACHE_FREE_PATTERN, cl,
3475 size);
3476 }
3477 }
3478
3479 MBUF_INIT(ms, 0, MT_FREE);
3480 if (class == MC_MBUF_16KCL) {
3481 MBUF_16KCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
3482 } else if (class == MC_MBUF_BIGCL) {
3483 MBUF_BIGCL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
3484 } else {
3485 MBUF_CL_INIT(ms, cl, rfa, 0, EXTF_COMPOSITE);
3486 }
3487 VERIFY(ms->m_flags == M_EXT);
3488 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
3489
3490 *list = (mcache_obj_t *)m;
3491 (*list)->obj_next = NULL;
3492 list = *plist = &(*list)->obj_next;
3493 }
3494
3495 fail:
3496 /*
3497 * Free up what's left of the above.
3498 */
3499 if (mp_list != NULL) {
3500 mcache_free_ext(m_cache(MC_MBUF), mp_list);
3501 }
3502 if (clp_list != NULL) {
3503 mcache_free_ext(m_cache(cl_class), clp_list);
3504 }
3505 if (ref_list != NULL) {
3506 mcache_free_ext(ref_cache, ref_list);
3507 }
3508
3509 lck_mtx_lock(mbuf_mlock);
3510 if (num > 0 || cnum > 0) {
3511 m_total(class) += cnum;
3512 VERIFY(m_total(class) <= m_maxlimit(class));
3513 m_alloc_cnt(class) += num + cnum;
3514 }
3515 if ((num + cnum) < want) {
3516 m_fail_cnt(class) += (want - (num + cnum));
3517 }
3518 lck_mtx_unlock(mbuf_mlock);
3519
3520 return num + cnum;
3521 }
3522
3523 /*
3524 * Common de-allocator for composite objects called by the CPU cache
3525 * layer when one or more elements need to be returned to the appropriate
3526 * global freelist.
3527 */
3528 static void
mbuf_cslab_free(void * arg,mcache_obj_t * list,int purged)3529 mbuf_cslab_free(void *arg, mcache_obj_t *list, int purged)
3530 {
3531 mbuf_class_t class = (mbuf_class_t)arg;
3532 unsigned int num;
3533 int w;
3534
3535 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
3536
3537 lck_mtx_lock(mbuf_mlock);
3538
3539 num = cslab_free(class, list, purged);
3540 m_free_cnt(class) += num;
3541
3542 if ((w = mb_waiters) > 0) {
3543 mb_waiters = 0;
3544 }
3545 if (w) {
3546 mbwdog_logger("waking up all threads");
3547 }
3548
3549 lck_mtx_unlock(mbuf_mlock);
3550
3551 if (w != 0) {
3552 wakeup(mb_waitchan);
3553 }
3554 }
3555
3556 /*
3557 * Common auditor for composite objects called by the CPU cache layer
3558 * during an allocation or free request. For the former, this is called
3559 * after the objects are obtained from either the bucket or slab layer
3560 * and before they are returned to the caller. For the latter, this is
3561 * called immediately during free and before placing the objects into
3562 * the bucket or slab layer.
3563 */
3564 static void
mbuf_cslab_audit(void * arg,mcache_obj_t * list,boolean_t alloc)3565 mbuf_cslab_audit(void *arg, mcache_obj_t *list, boolean_t alloc)
3566 {
3567 mbuf_class_t class = (mbuf_class_t)arg, cl_class;
3568 mcache_audit_t *mca;
3569 struct mbuf *m, *ms;
3570 mcl_slab_t *clsp, *nsp;
3571 size_t cl_size;
3572 void *cl;
3573
3574 ASSERT(MBUF_CLASS_VALID(class) && MBUF_CLASS_COMPOSITE(class));
3575 if (class == MC_MBUF_CL) {
3576 cl_class = MC_CL;
3577 } else if (class == MC_MBUF_BIGCL) {
3578 cl_class = MC_BIGCL;
3579 } else {
3580 cl_class = MC_16KCL;
3581 }
3582 cl_size = m_maxsize(cl_class);
3583
3584 while ((m = ms = (struct mbuf *)list) != NULL) {
3585 lck_mtx_lock(mbuf_mlock);
3586 /* Do the mbuf sanity checks and record its transaction */
3587 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
3588 mcl_audit_mbuf(mca, m, TRUE, alloc);
3589 if (mcltrace) {
3590 mcache_buffer_log(mca, m, m_cache(class), &mb_start);
3591 }
3592
3593 if (alloc) {
3594 mca->mca_uflags |= MB_COMP_INUSE;
3595 } else {
3596 mca->mca_uflags &= ~MB_COMP_INUSE;
3597 }
3598
3599 /*
3600 * Use the shadow mbuf in the audit structure if we are
3601 * freeing, since the contents of the actual mbuf has been
3602 * pattern-filled by the above call to mcl_audit_mbuf().
3603 */
3604 if (!alloc && mclverify) {
3605 ms = MCA_SAVED_MBUF_PTR(mca);
3606 }
3607
3608 /* Do the cluster sanity checks and record its transaction */
3609 cl = ms->m_ext.ext_buf;
3610 clsp = slab_get(cl);
3611 VERIFY(ms->m_flags == M_EXT && cl != NULL);
3612 VERIFY(m_get_rfa(ms) != NULL && MBUF_IS_COMPOSITE(ms));
3613 if (class == MC_MBUF_CL) {
3614 VERIFY(clsp->sl_refcnt >= 1 &&
3615 clsp->sl_refcnt <= NCLPG);
3616 } else {
3617 VERIFY(clsp->sl_refcnt >= 1 &&
3618 clsp->sl_refcnt <= NBCLPG);
3619 }
3620
3621 if (class == MC_MBUF_16KCL) {
3622 int k;
3623 for (nsp = clsp, k = 1; k < NSLABSP16KB; k++) {
3624 nsp = nsp->sl_next;
3625 /* Next slab must already be present */
3626 VERIFY(nsp != NULL);
3627 VERIFY(nsp->sl_refcnt == 1);
3628 }
3629 }
3630
3631
3632 mca = mcl_audit_buf2mca(cl_class, cl);
3633 mcl_audit_cluster(mca, cl, cl_size, alloc, FALSE);
3634 if (mcltrace) {
3635 mcache_buffer_log(mca, cl, m_cache(class), &mb_start);
3636 }
3637
3638 if (alloc) {
3639 mca->mca_uflags |= MB_COMP_INUSE;
3640 } else {
3641 mca->mca_uflags &= ~MB_COMP_INUSE;
3642 }
3643 lck_mtx_unlock(mbuf_mlock);
3644
3645 list = list->obj_next;
3646 }
3647 }
3648
3649 static void
m_vm_error_stats(uint32_t * cnt,uint64_t * ts,uint64_t * size,uint64_t alloc_size,kern_return_t error)3650 m_vm_error_stats(uint32_t *cnt, uint64_t *ts, uint64_t *size,
3651 uint64_t alloc_size, kern_return_t error)
3652 {
3653 *cnt = *cnt + 1;
3654 *ts = net_uptime();
3655 if (size) {
3656 *size = alloc_size;
3657 }
3658 switch (error) {
3659 case KERN_SUCCESS:
3660 break;
3661 case KERN_INVALID_ARGUMENT:
3662 mb_kmem_stats[0]++;
3663 break;
3664 case KERN_INVALID_ADDRESS:
3665 mb_kmem_stats[1]++;
3666 break;
3667 case KERN_RESOURCE_SHORTAGE:
3668 mb_kmem_stats[2]++;
3669 break;
3670 case KERN_NO_SPACE:
3671 mb_kmem_stats[3]++;
3672 break;
3673 case KERN_FAILURE:
3674 mb_kmem_stats[4]++;
3675 break;
3676 default:
3677 mb_kmem_stats[5]++;
3678 break;
3679 }
3680 }
3681
3682 static vm_offset_t
kmem_mb_alloc(vm_map_t mbmap,int size,int physContig,kern_return_t * err)3683 kmem_mb_alloc(vm_map_t mbmap, int size, int physContig, kern_return_t *err)
3684 {
3685 vm_offset_t addr = 0;
3686 kern_return_t kr = KERN_SUCCESS;
3687
3688 if (!physContig) {
3689 kr = kmem_alloc(mbmap, &addr, size,
3690 KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);
3691 } else {
3692 kr = kmem_alloc_contig(mbmap, &addr, size, PAGE_MASK, 0xfffff,
3693 0, KMA_KOBJECT | KMA_LOMEM, VM_KERN_MEMORY_MBUF);
3694 }
3695
3696 if (kr != KERN_SUCCESS) {
3697 addr = 0;
3698 }
3699 if (err) {
3700 *err = kr;
3701 }
3702
3703 return addr;
3704 }
3705
3706 /*
3707 * Allocate some number of mbuf clusters and place on cluster freelist.
3708 */
3709 static int
m_clalloc(const u_int32_t num,const int wait,const u_int32_t bufsize)3710 m_clalloc(const u_int32_t num, const int wait, const u_int32_t bufsize)
3711 {
3712 int i, count = 0;
3713 vm_size_t size = 0;
3714 int numpages = 0, large_buffer;
3715 vm_offset_t page = 0;
3716 mcache_audit_t *mca_list = NULL;
3717 mcache_obj_t *con_list = NULL;
3718 mcl_slab_t *sp;
3719 mbuf_class_t class;
3720 kern_return_t error;
3721
3722 /* Set if a buffer allocation needs allocation of multiple pages */
3723 large_buffer = ((bufsize == m_maxsize(MC_16KCL)) &&
3724 PAGE_SIZE < M16KCLBYTES);
3725 VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
3726 bufsize == m_maxsize(MC_16KCL));
3727
3728 VERIFY((bufsize == PAGE_SIZE) ||
3729 (bufsize > PAGE_SIZE && bufsize == m_maxsize(MC_16KCL)));
3730
3731 if (bufsize == m_size(MC_BIGCL)) {
3732 class = MC_BIGCL;
3733 } else {
3734 class = MC_16KCL;
3735 }
3736
3737 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3738
3739 /*
3740 * Multiple threads may attempt to populate the cluster map one
3741 * after another. Since we drop the lock below prior to acquiring
3742 * the physical page(s), our view of the cluster map may no longer
3743 * be accurate, and we could end up over-committing the pages beyond
3744 * the maximum allowed for each class. To prevent it, this entire
3745 * operation (including the page mapping) is serialized.
3746 */
3747 while (mb_clalloc_busy) {
3748 mb_clalloc_waiters++;
3749 (void) msleep(mb_clalloc_waitchan, mbuf_mlock,
3750 (PZERO - 1), "m_clalloc", NULL);
3751 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3752 }
3753
3754 /* We are busy now; tell everyone else to go away */
3755 mb_clalloc_busy = TRUE;
3756
3757 /*
3758 * Honor the caller's wish to block or not block. We have a way
3759 * to grow the pool asynchronously using the mbuf worker thread.
3760 */
3761 i = m_howmany(num, bufsize);
3762 if (i <= 0 || (wait & M_DONTWAIT)) {
3763 goto out;
3764 }
3765
3766 lck_mtx_unlock(mbuf_mlock);
3767
3768 size = round_page(i * bufsize);
3769 page = kmem_mb_alloc(mb_map, size, large_buffer, &error);
3770
3771 /*
3772 * If we did ask for "n" 16KB physically contiguous chunks
3773 * and didn't get them, then please try again without this
3774 * restriction.
3775 */
3776 net_update_uptime();
3777 if (large_buffer && page == 0) {
3778 m_vm_error_stats(&mb_kmem_contig_failed,
3779 &mb_kmem_contig_failed_ts,
3780 &mb_kmem_contig_failed_size,
3781 size, error);
3782 page = kmem_mb_alloc(mb_map, size, 0, &error);
3783 }
3784
3785 if (page == 0) {
3786 m_vm_error_stats(&mb_kmem_failed,
3787 &mb_kmem_failed_ts,
3788 &mb_kmem_failed_size,
3789 size, error);
3790 #if PAGE_SIZE == 4096
3791 if (bufsize == m_maxsize(MC_BIGCL)) {
3792 #else
3793 if (bufsize >= m_maxsize(MC_BIGCL)) {
3794 #endif
3795 /* Try for 1 page if failed */
3796 size = PAGE_SIZE;
3797 page = kmem_mb_alloc(mb_map, size, 0, &error);
3798 if (page == 0) {
3799 m_vm_error_stats(&mb_kmem_one_failed,
3800 &mb_kmem_one_failed_ts,
3801 NULL, size, error);
3802 }
3803 }
3804
3805 if (page == 0) {
3806 lck_mtx_lock(mbuf_mlock);
3807 goto out;
3808 }
3809 }
3810
3811 VERIFY(IS_P2ALIGNED(page, PAGE_SIZE));
3812 numpages = size / PAGE_SIZE;
3813
3814 /* If auditing is enabled, allocate the audit structures now */
3815 if (mclaudit != NULL) {
3816 int needed;
3817
3818 /*
3819 * Yes, I realize this is a waste of memory for clusters
3820 * that never get transformed into mbufs, as we may end
3821 * up with NMBPG-1 unused audit structures per cluster.
3822 * But doing so tremendously simplifies the allocation
3823 * strategy, since at this point we are not holding the
3824 * mbuf lock and the caller is okay to be blocked.
3825 */
3826 if (bufsize == PAGE_SIZE) {
3827 needed = numpages * NMBPG;
3828
3829 i = mcache_alloc_ext(mcl_audit_con_cache,
3830 &con_list, needed, MCR_SLEEP);
3831
3832 VERIFY(con_list != NULL && i == needed);
3833 } else {
3834 /*
3835 * if multiple 4K pages are being used for a
3836 * 16K cluster
3837 */
3838 needed = numpages / NSLABSP16KB;
3839 }
3840
3841 i = mcache_alloc_ext(mcache_audit_cache,
3842 (mcache_obj_t **)&mca_list, needed, MCR_SLEEP);
3843
3844 VERIFY(mca_list != NULL && i == needed);
3845 }
3846
3847 lck_mtx_lock(mbuf_mlock);
3848
3849 for (i = 0; i < numpages; i++, page += PAGE_SIZE) {
3850 ppnum_t offset =
3851 ((unsigned char *)page - mbutl) >> PAGE_SHIFT;
3852 ppnum_t new_page = pmap_find_phys(kernel_pmap, page);
3853
3854 /*
3855 * If there is a mapper the appropriate I/O page is
3856 * returned; zero out the page to discard its past
3857 * contents to prevent exposing leftover kernel memory.
3858 */
3859 VERIFY(offset < mcl_pages);
3860 if (mcl_paddr_base != 0) {
3861 bzero((void *)(uintptr_t) page, PAGE_SIZE);
3862 new_page = IOMapperInsertPage(mcl_paddr_base,
3863 offset, new_page);
3864 }
3865 mcl_paddr[offset] = new_page;
3866
3867 /* Pattern-fill this fresh page */
3868 if (mclverify) {
3869 mcache_set_pattern(MCACHE_FREE_PATTERN,
3870 (caddr_t)page, PAGE_SIZE);
3871 }
3872 if (bufsize == PAGE_SIZE) {
3873 mcache_obj_t *buf;
3874 /* One for the entire page */
3875 sp = slab_get((void *)page);
3876 if (mclaudit != NULL) {
3877 mcl_audit_init((void *)page,
3878 &mca_list, &con_list,
3879 AUDIT_CONTENTS_SIZE, NMBPG);
3880 }
3881 VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
3882 slab_init(sp, class, SLF_MAPPED, (void *)page,
3883 (void *)page, PAGE_SIZE, 0, 1);
3884 buf = (mcache_obj_t *)page;
3885 buf->obj_next = NULL;
3886
3887 /* Insert this slab */
3888 slab_insert(sp, class);
3889
3890 /* Update stats now since slab_get drops the lock */
3891 ++m_infree(class);
3892 ++m_total(class);
3893 VERIFY(m_total(class) <= m_maxlimit(class));
3894 if (class == MC_BIGCL) {
3895 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
3896 m_infree(MC_MBUF_BIGCL);
3897 mbstat.m_bigclusters = m_total(MC_BIGCL);
3898 }
3899 ++count;
3900 } else if ((bufsize > PAGE_SIZE) &&
3901 (i % NSLABSP16KB) == 0) {
3902 union m16kcluster *m16kcl = (union m16kcluster *)page;
3903 mcl_slab_t *nsp;
3904 int k;
3905
3906 /* One for the entire 16KB */
3907 sp = slab_get(m16kcl);
3908 if (mclaudit != NULL) {
3909 mcl_audit_init(m16kcl, &mca_list, NULL, 0, 1);
3910 }
3911
3912 VERIFY(sp->sl_refcnt == 0 && sp->sl_flags == 0);
3913 slab_init(sp, MC_16KCL, SLF_MAPPED,
3914 m16kcl, m16kcl, bufsize, 0, 1);
3915 m16kcl->m16kcl_next = NULL;
3916
3917 /*
3918 * 2nd-Nth page's slab is part of the first one,
3919 * where N is NSLABSP16KB.
3920 */
3921 for (k = 1; k < NSLABSP16KB; k++) {
3922 nsp = slab_get(((union mbigcluster *)page) + k);
3923 VERIFY(nsp->sl_refcnt == 0 &&
3924 nsp->sl_flags == 0);
3925 slab_init(nsp, MC_16KCL,
3926 SLF_MAPPED | SLF_PARTIAL,
3927 m16kcl, NULL, 0, 0, 0);
3928 }
3929 /* Insert this slab */
3930 slab_insert(sp, MC_16KCL);
3931
3932 /* Update stats now since slab_get drops the lock */
3933 ++m_infree(MC_16KCL);
3934 ++m_total(MC_16KCL);
3935 VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
3936 ++count;
3937 }
3938 }
3939 VERIFY(mca_list == NULL && con_list == NULL);
3940
3941 /* We're done; let others enter */
3942 mb_clalloc_busy = FALSE;
3943 if (mb_clalloc_waiters > 0) {
3944 mb_clalloc_waiters = 0;
3945 wakeup(mb_clalloc_waitchan);
3946 }
3947
3948 return count;
3949 out:
3950 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
3951
3952 mtracelarge_register(size);
3953
3954 /* We're done; let others enter */
3955 mb_clalloc_busy = FALSE;
3956 if (mb_clalloc_waiters > 0) {
3957 mb_clalloc_waiters = 0;
3958 wakeup(mb_clalloc_waitchan);
3959 }
3960
3961 /*
3962 * When non-blocking we kick a thread if we have to grow the
3963 * pool or if the number of free clusters is less than requested.
3964 */
3965 if (i > 0 && mbuf_worker_ready && mbuf_worker_needs_wakeup) {
3966 mbwdog_logger("waking up the worker thread to to grow %s by %d",
3967 m_cname(class), i);
3968 wakeup((caddr_t)&mbuf_worker_needs_wakeup);
3969 mbuf_worker_needs_wakeup = FALSE;
3970 }
3971 if (class == MC_BIGCL) {
3972 if (i > 0) {
3973 /*
3974 * Remember total number of 4KB clusters needed
3975 * at this time.
3976 */
3977 i += m_total(MC_BIGCL);
3978 if (i > m_region_expand(MC_BIGCL)) {
3979 m_region_expand(MC_BIGCL) = i;
3980 }
3981 }
3982 if (m_infree(MC_BIGCL) >= num) {
3983 return 1;
3984 }
3985 } else {
3986 if (i > 0) {
3987 /*
3988 * Remember total number of 16KB clusters needed
3989 * at this time.
3990 */
3991 i += m_total(MC_16KCL);
3992 if (i > m_region_expand(MC_16KCL)) {
3993 m_region_expand(MC_16KCL) = i;
3994 }
3995 }
3996 if (m_infree(MC_16KCL) >= num) {
3997 return 1;
3998 }
3999 }
4000 return 0;
4001 }
4002
4003 /*
4004 * Populate the global freelist of the corresponding buffer class.
4005 */
4006 static int
4007 freelist_populate(mbuf_class_t class, unsigned int num, int wait)
4008 {
4009 mcache_obj_t *o = NULL;
4010 int i, numpages = 0, count;
4011 mbuf_class_t super_class;
4012
4013 VERIFY(class == MC_MBUF || class == MC_CL || class == MC_BIGCL ||
4014 class == MC_16KCL);
4015
4016 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
4017
4018 VERIFY(PAGE_SIZE == m_maxsize(MC_BIGCL) ||
4019 PAGE_SIZE == m_maxsize(MC_16KCL));
4020
4021 if (m_maxsize(class) >= PAGE_SIZE) {
4022 return m_clalloc(num, wait, m_maxsize(class)) != 0;
4023 }
4024
4025 /*
4026 * The rest of the function will allocate pages and will slice
4027 * them up into the right size
4028 */
4029
4030 numpages = (num * m_size(class) + PAGE_SIZE - 1) / PAGE_SIZE;
4031
4032 /* Currently assume that pages are 4K or 16K */
4033 if (PAGE_SIZE == m_maxsize(MC_BIGCL)) {
4034 super_class = MC_BIGCL;
4035 } else {
4036 super_class = MC_16KCL;
4037 }
4038
4039 i = m_clalloc(numpages, wait, m_maxsize(super_class));
4040
4041 /* how many objects will we cut the page into? */
4042 int numobj = PAGE_SIZE / m_maxsize(class);
4043
4044 for (count = 0; count < numpages; count++) {
4045 /* respect totals, minlimit, maxlimit */
4046 if (m_total(super_class) <= m_minlimit(super_class) ||
4047 m_total(class) >= m_maxlimit(class)) {
4048 break;
4049 }
4050
4051 if ((o = slab_alloc(super_class, wait)) == NULL) {
4052 break;
4053 }
4054
4055 struct mbuf *m = (struct mbuf *)o;
4056 union mcluster *c = (union mcluster *)o;
4057 union mbigcluster *mbc = (union mbigcluster *)o;
4058 mcl_slab_t *sp = slab_get(o);
4059 mcache_audit_t *mca = NULL;
4060
4061 /*
4062 * since one full page will be converted to MC_MBUF or
4063 * MC_CL, verify that the reference count will match that
4064 * assumption
4065 */
4066 VERIFY(sp->sl_refcnt == 1 && slab_is_detached(sp));
4067 VERIFY((sp->sl_flags & (SLF_MAPPED | SLF_PARTIAL)) == SLF_MAPPED);
4068 /*
4069 * Make sure that the cluster is unmolested
4070 * while in freelist
4071 */
4072 if (mclverify) {
4073 mca = mcl_audit_buf2mca(super_class,
4074 (mcache_obj_t *)o);
4075 mcache_audit_free_verify(mca,
4076 (mcache_obj_t *)o, 0, m_maxsize(super_class));
4077 }
4078
4079 /* Reinitialize it as an mbuf or 2K or 4K slab */
4080 slab_init(sp, class, sp->sl_flags,
4081 sp->sl_base, NULL, PAGE_SIZE, 0, numobj);
4082
4083 VERIFY(sp->sl_head == NULL);
4084
4085 VERIFY(m_total(super_class) >= 1);
4086 m_total(super_class)--;
4087
4088 if (super_class == MC_BIGCL) {
4089 mbstat.m_bigclusters = m_total(MC_BIGCL);
4090 }
4091
4092 m_total(class) += numobj;
4093 VERIFY(m_total(class) <= m_maxlimit(class));
4094 m_infree(class) += numobj;
4095
4096 i = numobj;
4097 if (class == MC_MBUF) {
4098 mbstat.m_mbufs = m_total(MC_MBUF);
4099 mtype_stat_add(MT_FREE, NMBPG);
4100 while (i--) {
4101 /*
4102 * If auditing is enabled, construct the
4103 * shadow mbuf in the audit structure
4104 * instead of the actual one.
4105 * mbuf_slab_audit() will take care of
4106 * restoring the contents after the
4107 * integrity check.
4108 */
4109 if (mclaudit != NULL) {
4110 struct mbuf *ms;
4111 mca = mcl_audit_buf2mca(MC_MBUF,
4112 (mcache_obj_t *)m);
4113 ms = MCA_SAVED_MBUF_PTR(mca);
4114 ms->m_type = MT_FREE;
4115 } else {
4116 m->m_type = MT_FREE;
4117 }
4118 m->m_next = sp->sl_head;
4119 sp->sl_head = (void *)m++;
4120 }
4121 } else if (class == MC_CL) { /* MC_CL */
4122 mbstat.m_clfree =
4123 m_infree(MC_CL) + m_infree(MC_MBUF_CL);
4124 mbstat.m_clusters = m_total(MC_CL);
4125 while (i--) {
4126 c->mcl_next = sp->sl_head;
4127 sp->sl_head = (void *)c++;
4128 }
4129 } else {
4130 VERIFY(class == MC_BIGCL);
4131 mbstat.m_bigclusters = m_total(MC_BIGCL);
4132 mbstat.m_bigclfree = m_infree(MC_BIGCL) +
4133 m_infree(MC_MBUF_BIGCL);
4134 while (i--) {
4135 mbc->mbc_next = sp->sl_head;
4136 sp->sl_head = (void *)mbc++;
4137 }
4138 }
4139
4140 /* Insert into the mbuf or 2k or 4k slab list */
4141 slab_insert(sp, class);
4142
4143 if ((i = mb_waiters) > 0) {
4144 mb_waiters = 0;
4145 }
4146 if (i != 0) {
4147 mbwdog_logger("waking up all threads");
4148 wakeup(mb_waitchan);
4149 }
4150 }
4151 return count != 0;
4152 }
4153
4154 /*
4155 * For each class, initialize the freelist to hold m_minlimit() objects.
4156 */
4157 static void
4158 freelist_init(mbuf_class_t class)
4159 {
4160 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
4161
4162 VERIFY(class == MC_CL || class == MC_BIGCL);
4163 VERIFY(m_total(class) == 0);
4164 VERIFY(m_minlimit(class) > 0);
4165
4166 while (m_total(class) < m_minlimit(class)) {
4167 (void) freelist_populate(class, m_minlimit(class), M_WAIT);
4168 }
4169
4170 VERIFY(m_total(class) >= m_minlimit(class));
4171 }
4172
4173 /*
4174 * (Inaccurately) check if it might be worth a trip back to the
4175 * mcache layer due the availability of objects there. We'll
4176 * end up back here if there's nothing up there.
4177 */
4178 static boolean_t
4179 mbuf_cached_above(mbuf_class_t class, int wait)
4180 {
4181 switch (class) {
4182 case MC_MBUF:
4183 if (wait & MCR_COMP) {
4184 return !mcache_bkt_isempty(m_cache(MC_MBUF_CL)) ||
4185 !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL));
4186 }
4187 break;
4188
4189 case MC_CL:
4190 if (wait & MCR_COMP) {
4191 return !mcache_bkt_isempty(m_cache(MC_MBUF_CL));
4192 }
4193 break;
4194
4195 case MC_BIGCL:
4196 if (wait & MCR_COMP) {
4197 return !mcache_bkt_isempty(m_cache(MC_MBUF_BIGCL));
4198 }
4199 break;
4200
4201 case MC_16KCL:
4202 if (wait & MCR_COMP) {
4203 return !mcache_bkt_isempty(m_cache(MC_MBUF_16KCL));
4204 }
4205 break;
4206
4207 case MC_MBUF_CL:
4208 case MC_MBUF_BIGCL:
4209 case MC_MBUF_16KCL:
4210 break;
4211
4212 default:
4213 VERIFY(0);
4214 /* NOTREACHED */
4215 }
4216
4217 return !mcache_bkt_isempty(m_cache(class));
4218 }
4219
4220 /*
4221 * If possible, convert constructed objects to raw ones.
4222 */
4223 static boolean_t
4224 mbuf_steal(mbuf_class_t class, unsigned int num)
4225 {
4226 mcache_obj_t *top = NULL;
4227 mcache_obj_t **list = ⊤
4228 unsigned int tot = 0;
4229
4230 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
4231
4232 switch (class) {
4233 case MC_MBUF:
4234 case MC_CL:
4235 case MC_BIGCL:
4236 case MC_16KCL:
4237 return FALSE;
4238
4239 case MC_MBUF_CL:
4240 case MC_MBUF_BIGCL:
4241 case MC_MBUF_16KCL:
4242 /* Get the required number of constructed objects if possible */
4243 if (m_infree(class) > m_minlimit(class)) {
4244 tot = cslab_alloc(class, &list,
4245 MIN(num, m_infree(class)));
4246 }
4247
4248 /* And destroy them to get back the raw objects */
4249 if (top != NULL) {
4250 (void) cslab_free(class, top, 1);
4251 }
4252 break;
4253
4254 default:
4255 VERIFY(0);
4256 /* NOTREACHED */
4257 }
4258
4259 return tot == num;
4260 }
4261
4262 static void
4263 m_reclaim(mbuf_class_t class, unsigned int num, boolean_t comp)
4264 {
4265 int m, bmap = 0;
4266
4267 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
4268
4269 VERIFY(m_total(MC_CL) <= m_maxlimit(MC_CL));
4270 VERIFY(m_total(MC_BIGCL) <= m_maxlimit(MC_BIGCL));
4271 VERIFY(m_total(MC_16KCL) <= m_maxlimit(MC_16KCL));
4272
4273 /*
4274 * This logic can be made smarter; for now, simply mark
4275 * all other related classes as potential victims.
4276 */
4277 switch (class) {
4278 case MC_MBUF:
4279 m_wantpurge(MC_CL)++;
4280 m_wantpurge(MC_BIGCL)++;
4281 m_wantpurge(MC_MBUF_CL)++;
4282 m_wantpurge(MC_MBUF_BIGCL)++;
4283 break;
4284
4285 case MC_CL:
4286 m_wantpurge(MC_MBUF)++;
4287 m_wantpurge(MC_BIGCL)++;
4288 m_wantpurge(MC_MBUF_BIGCL)++;
4289 if (!comp) {
4290 m_wantpurge(MC_MBUF_CL)++;
4291 }
4292 break;
4293
4294 case MC_BIGCL:
4295 m_wantpurge(MC_MBUF)++;
4296 m_wantpurge(MC_CL)++;
4297 m_wantpurge(MC_MBUF_CL)++;
4298 if (!comp) {
4299 m_wantpurge(MC_MBUF_BIGCL)++;
4300 }
4301 break;
4302
4303 case MC_16KCL:
4304 if (!comp) {
4305 m_wantpurge(MC_MBUF_16KCL)++;
4306 }
4307 break;
4308
4309 default:
4310 VERIFY(0);
4311 /* NOTREACHED */
4312 }
4313
4314 /*
4315 * Run through each marked class and check if we really need to
4316 * purge (and therefore temporarily disable) the per-CPU caches
4317 * layer used by the class. If so, remember the classes since
4318 * we are going to drop the lock below prior to purging.
4319 */
4320 for (m = 0; m < NELEM(mbuf_table); m++) {
4321 if (m_wantpurge(m) > 0) {
4322 m_wantpurge(m) = 0;
4323 /*
4324 * Try hard to steal the required number of objects
4325 * from the freelist of other mbuf classes. Only
4326 * purge and disable the per-CPU caches layer when
4327 * we don't have enough; it's the last resort.
4328 */
4329 if (!mbuf_steal(m, num)) {
4330 bmap |= (1 << m);
4331 }
4332 }
4333 }
4334
4335 lck_mtx_unlock(mbuf_mlock);
4336
4337 if (bmap != 0) {
4338 /* signal the domains to drain */
4339 net_drain_domains();
4340
4341 /* Sigh; we have no other choices but to ask mcache to purge */
4342 for (m = 0; m < NELEM(mbuf_table); m++) {
4343 if ((bmap & (1 << m)) &&
4344 mcache_purge_cache(m_cache(m), TRUE)) {
4345 lck_mtx_lock(mbuf_mlock);
4346 m_purge_cnt(m)++;
4347 mbstat.m_drain++;
4348 lck_mtx_unlock(mbuf_mlock);
4349 }
4350 }
4351 } else {
4352 /*
4353 * Request mcache to reap extra elements from all of its caches;
4354 * note that all reaps are serialized and happen only at a fixed
4355 * interval.
4356 */
4357 mcache_reap();
4358 }
4359 lck_mtx_lock(mbuf_mlock);
4360 }
4361 #endif /* CONFIG_MBUF_MCACHE */
4362
4363 static inline struct mbuf *
4364 m_get_common(int wait, short type, int hdr)
4365 {
4366 struct mbuf *m;
4367
4368 #if CONFIG_MBUF_MCACHE
4369 int mcflags = MSLEEPF(wait);
4370
4371 /* Is this due to a non-blocking retry? If so, then try harder */
4372 if (mcflags & MCR_NOSLEEP) {
4373 mcflags |= MCR_TRYHARD;
4374 }
4375
4376 m = mcache_alloc(m_cache(MC_MBUF), mcflags);
4377 #else
4378 m = mz_alloc(wait);
4379 #endif /* CONFIG_MBUF_MCACHE */
4380 if (m != NULL) {
4381 MBUF_INIT(m, hdr, type);
4382 mtype_stat_inc(type);
4383 mtype_stat_dec(MT_FREE);
4384 }
4385 return m;
4386 }
4387
4388 /*
4389 * Space allocation routines; these are also available as macros
4390 * for critical paths.
4391 */
4392 #define _M_GET(wait, type) m_get_common(wait, type, 0)
4393 #define _M_GETHDR(wait, type) m_get_common(wait, type, 1)
4394 #define _M_RETRY(wait, type) _M_GET(wait, type)
4395 #define _M_RETRYHDR(wait, type) _M_GETHDR(wait, type)
4396 #define _MGET(m, how, type) ((m) = _M_GET(how, type))
4397 #define _MGETHDR(m, how, type) ((m) = _M_GETHDR(how, type))
4398
4399 struct mbuf *
4400 m_get(int wait, int type)
4401 {
4402 return _M_GET(wait, type);
4403 }
4404
4405 struct mbuf *
4406 m_gethdr(int wait, int type)
4407 {
4408 return _M_GETHDR(wait, type);
4409 }
4410
4411 struct mbuf *
4412 m_retry(int wait, int type)
4413 {
4414 return _M_RETRY(wait, type);
4415 }
4416
4417 struct mbuf *
4418 m_retryhdr(int wait, int type)
4419 {
4420 return _M_RETRYHDR(wait, type);
4421 }
4422
4423 struct mbuf *
4424 m_getclr(int wait, int type)
4425 {
4426 struct mbuf *m;
4427
4428 _MGET(m, wait, type);
4429 if (m != NULL) {
4430 bzero(MTOD(m, caddr_t), MLEN);
4431 }
4432 return m;
4433 }
4434
4435 static int
4436 m_free_paired(struct mbuf *m)
4437 {
4438 VERIFY((m->m_flags & M_EXT) && (MEXT_FLAGS(m) & EXTF_PAIRED));
4439
4440 os_atomic_thread_fence(seq_cst);
4441 if (MEXT_PMBUF(m) == m) {
4442 /*
4443 * Paired ref count might be negative in case we lose
4444 * against another thread clearing MEXT_PMBUF, in the
4445 * event it occurs after the above memory barrier sync.
4446 * In that case just ignore as things have been unpaired.
4447 */
4448 int16_t prefcnt = os_atomic_dec(&MEXT_PREF(m), acq_rel);
4449 if (prefcnt > 1) {
4450 return 1;
4451 } else if (prefcnt == 1) {
4452 m_ext_free_func_t m_free_func = m_get_ext_free(m);
4453 VERIFY(m_free_func != NULL);
4454 (*m_free_func)(m->m_ext.ext_buf,
4455 m->m_ext.ext_size, m_get_ext_arg(m));
4456 return 1;
4457 } else if (prefcnt == 0) {
4458 VERIFY(MBUF_IS_PAIRED(m));
4459
4460 /*
4461 * Restore minref to its natural value, so that
4462 * the caller will be able to free the cluster
4463 * as appropriate.
4464 */
4465 MEXT_MINREF(m) = 0;
4466
4467 /*
4468 * Clear MEXT_PMBUF, but leave EXTF_PAIRED intact
4469 * as it is immutable. atomic_set_ptr also causes
4470 * memory barrier sync.
4471 */
4472 os_atomic_store(&MEXT_PMBUF(m), NULL, release);
4473
4474 switch (m->m_ext.ext_size) {
4475 case MCLBYTES:
4476 m_set_ext(m, m_get_rfa(m), NULL, NULL);
4477 break;
4478
4479 case MBIGCLBYTES:
4480 m_set_ext(m, m_get_rfa(m), m_bigfree, NULL);
4481 break;
4482
4483 case M16KCLBYTES:
4484 m_set_ext(m, m_get_rfa(m), m_16kfree, NULL);
4485 break;
4486
4487 default:
4488 VERIFY(0);
4489 /* NOTREACHED */
4490 }
4491 }
4492 }
4493
4494 /*
4495 * Tell caller the unpair has occurred, and that the reference
4496 * count on the external cluster held for the paired mbuf should
4497 * now be dropped.
4498 */
4499 return 0;
4500 }
4501
4502 struct mbuf *
4503 m_free(struct mbuf *m)
4504 {
4505 struct mbuf *n = m->m_next;
4506
4507 if (m->m_type == MT_FREE) {
4508 panic("m_free: freeing an already freed mbuf");
4509 }
4510
4511 if (m->m_flags & M_PKTHDR) {
4512 /* Check for scratch area overflow */
4513 m_redzone_verify(m);
4514 /* Free the aux data and tags if there is any */
4515 m_tag_delete_chain(m);
4516
4517 m_do_tx_compl_callback(m, NULL);
4518 }
4519
4520 if (m->m_flags & M_EXT) {
4521 if (MBUF_IS_PAIRED(m) && m_free_paired(m)) {
4522 return n;
4523 }
4524 /*
4525 * Make sure that we don't touch any ext_ref
4526 * member after we decrement the reference count
4527 * since that may lead to use-after-free
4528 * when we do not hold the last reference.
4529 */
4530 const bool composite = !!(MEXT_FLAGS(m) & EXTF_COMPOSITE);
4531 const m_ext_free_func_t m_free_func = m_get_ext_free(m);
4532 const uint16_t minref = MEXT_MINREF(m);
4533 const uint16_t refcnt = m_decref(m);
4534
4535 if (refcnt == minref && !composite) {
4536 #if CONFIG_MBUF_MCACHE
4537 if (m_free_func == NULL) {
4538 mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
4539 } else if (m_free_func == m_bigfree) {
4540 mcache_free(m_cache(MC_BIGCL),
4541 m->m_ext.ext_buf);
4542 } else if (m_free_func == m_16kfree) {
4543 mcache_free(m_cache(MC_16KCL),
4544 m->m_ext.ext_buf);
4545 } else {
4546 (*m_free_func)(m->m_ext.ext_buf,
4547 m->m_ext.ext_size, m_get_ext_arg(m));
4548 }
4549 mcache_free(ref_cache, m_get_rfa(m));
4550 #else
4551 if (m_free_func == NULL) {
4552 mz_cl_free(ZONE_ID_CLUSTER_2K, m->m_ext.ext_buf);
4553 } else if (m_free_func == m_bigfree) {
4554 mz_cl_free(ZONE_ID_CLUSTER_4K, m->m_ext.ext_buf);
4555 } else if (m_free_func == m_16kfree) {
4556 mz_cl_free(ZONE_ID_CLUSTER_16K, m->m_ext.ext_buf);
4557 } else {
4558 (*m_free_func)(m->m_ext.ext_buf,
4559 m->m_ext.ext_size, m_get_ext_arg(m));
4560 }
4561 mz_ref_free(m_get_rfa(m));
4562 #endif /* CONFIG_MBUF_MCACHE */
4563 m_set_ext(m, NULL, NULL, NULL);
4564 } else if (refcnt == minref && composite) {
4565 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
4566
4567 mtype_stat_dec(m->m_type);
4568 mtype_stat_inc(MT_FREE);
4569
4570 m->m_type = MT_FREE;
4571 m->m_flags = M_EXT;
4572 m->m_len = 0;
4573 m->m_next = m->m_nextpkt = NULL;
4574 /*
4575 * MEXT_FLAGS is safe to access here
4576 * since we are now sure that we held
4577 * the last reference to ext_ref.
4578 */
4579 MEXT_FLAGS(m) &= ~EXTF_READONLY;
4580
4581 #if CONFIG_MBUF_MCACHE
4582 /* "Free" into the intermediate cache */
4583 if (m_free_func == NULL) {
4584 mcache_free(m_cache(MC_MBUF_CL), m);
4585 } else if (m_free_func == m_bigfree) {
4586 mcache_free(m_cache(MC_MBUF_BIGCL), m);
4587 } else {
4588 VERIFY(m_free_func == m_16kfree);
4589 mcache_free(m_cache(MC_MBUF_16KCL), m);
4590 }
4591 #else
4592 /* "Free" into the intermediate cache */
4593 if (m_free_func == NULL) {
4594 mz_composite_free(MC_MBUF_CL, m);
4595 } else if (m_free_func == m_bigfree) {
4596 mz_composite_free(MC_MBUF_BIGCL, m);
4597 } else {
4598 VERIFY(m_free_func == m_16kfree);
4599 mz_composite_free(MC_MBUF_16KCL, m);
4600 }
4601 #endif /* CONFIG_MBUF_MCACHE */
4602 return n;
4603 }
4604 }
4605
4606 mtype_stat_dec(m->m_type);
4607 mtype_stat_inc(MT_FREE);
4608
4609 m->m_type = MT_FREE;
4610 m->m_flags = m->m_len = 0;
4611 m->m_next = m->m_nextpkt = NULL;
4612
4613 #if CONFIG_MBUF_MCACHE
4614 mcache_free(m_cache(MC_MBUF), m);
4615 #else
4616 mz_free(m);
4617 #endif /* CONFIG_MBUF_MCACHE */
4618
4619 return n;
4620 }
4621
4622 __private_extern__ struct mbuf *
4623 m_clattach(struct mbuf *m, int type, caddr_t extbuf,
4624 void (*extfree)(caddr_t, u_int, caddr_t), size_t extsize, caddr_t extarg,
4625 int wait, int pair)
4626 {
4627 struct ext_ref *rfa = NULL;
4628
4629 /*
4630 * If pairing is requested and an existing mbuf is provided, reject
4631 * it if it's already been paired to another cluster. Otherwise,
4632 * allocate a new one or free any existing below.
4633 */
4634 if ((m != NULL && MBUF_IS_PAIRED(m)) ||
4635 (m == NULL && (m = _M_GETHDR(wait, type)) == NULL)) {
4636 return NULL;
4637 }
4638
4639 if (m->m_flags & M_EXT) {
4640 /*
4641 * Make sure that we don't touch any ext_ref
4642 * member after we decrement the reference count
4643 * since that may lead to use-after-free
4644 * when we do not hold the last reference.
4645 */
4646 const bool composite = !!(MEXT_FLAGS(m) & EXTF_COMPOSITE);
4647 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED) && MEXT_PMBUF(m) == NULL);
4648 const m_ext_free_func_t m_free_func = m_get_ext_free(m);
4649 const uint16_t minref = MEXT_MINREF(m);
4650 const uint16_t refcnt = m_decref(m);
4651
4652 if (refcnt == minref && !composite) {
4653 #if CONFIG_MBUF_MCACHE
4654 if (m_free_func == NULL) {
4655 mcache_free(m_cache(MC_CL), m->m_ext.ext_buf);
4656 } else if (m_free_func == m_bigfree) {
4657 mcache_free(m_cache(MC_BIGCL),
4658 m->m_ext.ext_buf);
4659 } else if (m_free_func == m_16kfree) {
4660 mcache_free(m_cache(MC_16KCL),
4661 m->m_ext.ext_buf);
4662 } else {
4663 (*m_free_func)(m->m_ext.ext_buf,
4664 m->m_ext.ext_size, m_get_ext_arg(m));
4665 }
4666 #else
4667 if (m_free_func == NULL) {
4668 mz_cl_free(ZONE_ID_CLUSTER_2K, m->m_ext.ext_buf);
4669 } else if (m_free_func == m_bigfree) {
4670 mz_cl_free(ZONE_ID_CLUSTER_4K, m->m_ext.ext_buf);
4671 } else if (m_free_func == m_16kfree) {
4672 mz_cl_free(ZONE_ID_CLUSTER_16K, m->m_ext.ext_buf);
4673 } else {
4674 (*m_free_func)(m->m_ext.ext_buf,
4675 m->m_ext.ext_size, m_get_ext_arg(m));
4676 }
4677 #endif /* CONFIG_MBUF_MCACHE */
4678 /* Re-use the reference structure */
4679 rfa = m_get_rfa(m);
4680 } else if (refcnt == minref && composite) {
4681 VERIFY(m->m_type != MT_FREE);
4682
4683 mtype_stat_dec(m->m_type);
4684 mtype_stat_inc(MT_FREE);
4685
4686 m->m_type = MT_FREE;
4687 m->m_flags = M_EXT;
4688 m->m_len = 0;
4689 m->m_next = m->m_nextpkt = NULL;
4690
4691 /*
4692 * MEXT_FLAGS is safe to access here
4693 * since we are now sure that we held
4694 * the last reference to ext_ref.
4695 */
4696 MEXT_FLAGS(m) &= ~EXTF_READONLY;
4697
4698 /* "Free" into the intermediate cache */
4699 #if CONFIG_MBUF_MCACHE
4700 if (m_free_func == NULL) {
4701 mcache_free(m_cache(MC_MBUF_CL), m);
4702 } else if (m_free_func == m_bigfree) {
4703 mcache_free(m_cache(MC_MBUF_BIGCL), m);
4704 } else {
4705 VERIFY(m_free_func == m_16kfree);
4706 mcache_free(m_cache(MC_MBUF_16KCL), m);
4707 }
4708 #else
4709 if (m_free_func == NULL) {
4710 mz_composite_free(MC_MBUF_CL, m);
4711 } else if (m_free_func == m_bigfree) {
4712 mz_composite_free(MC_MBUF_BIGCL, m);
4713 } else {
4714 VERIFY(m_free_func == m_16kfree);
4715 mz_composite_free(MC_MBUF_16KCL, m);
4716 }
4717 #endif /* CONFIG_MBUF_MCACHE */
4718 /*
4719 * Allocate a new mbuf, since we didn't divorce
4720 * the composite mbuf + cluster pair above.
4721 */
4722 if ((m = _M_GETHDR(wait, type)) == NULL) {
4723 return NULL;
4724 }
4725 }
4726 }
4727
4728 #if CONFIG_MBUF_MCACHE
4729 if (rfa == NULL &&
4730 (rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
4731 m_free(m);
4732 return NULL;
4733 }
4734 #else
4735 if (rfa == NULL &&
4736 (rfa = mz_ref_alloc(wait)) == NULL) {
4737 m_free(m);
4738 return NULL;
4739 }
4740 #endif /* CONFIG_MBUF_MCACHE */
4741
4742 if (!pair) {
4743 MEXT_INIT(m, extbuf, extsize, extfree, extarg, rfa,
4744 0, 1, 0, 0, 0, NULL);
4745 } else {
4746 MEXT_INIT(m, extbuf, extsize, extfree, (caddr_t)m, rfa,
4747 1, 1, 1, EXTF_PAIRED, 0, m);
4748 }
4749
4750 return m;
4751 }
4752
4753 /*
4754 * Perform `fast' allocation mbuf clusters from a cache of recently-freed
4755 * clusters. (If the cache is empty, new clusters are allocated en-masse.)
4756 */
4757 struct mbuf *
4758 m_getcl(int wait, int type, int flags)
4759 {
4760 struct mbuf *m = NULL;
4761 int hdr = (flags & M_PKTHDR);
4762
4763 #if CONFIG_MBUF_MCACHE
4764 int mcflags = MSLEEPF(wait);
4765
4766 /* Is this due to a non-blocking retry? If so, then try harder */
4767 if (mcflags & MCR_NOSLEEP) {
4768 mcflags |= MCR_TRYHARD;
4769 }
4770
4771 m = mcache_alloc(m_cache(MC_MBUF_CL), mcflags);
4772 #else
4773 m = mz_composite_alloc(MC_MBUF_CL, wait);
4774 #endif /* CONFIG_MBUF_MCACHE */
4775 if (m != NULL) {
4776 u_int16_t flag;
4777 struct ext_ref *rfa;
4778 void *cl;
4779
4780 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
4781 cl = m->m_ext.ext_buf;
4782 rfa = m_get_rfa(m);
4783
4784 ASSERT(cl != NULL && rfa != NULL);
4785 VERIFY(MBUF_IS_COMPOSITE(m) && m_get_ext_free(m) == NULL);
4786
4787 flag = MEXT_FLAGS(m);
4788
4789 MBUF_INIT(m, hdr, type);
4790 MBUF_CL_INIT(m, cl, rfa, 1, flag);
4791
4792 mtype_stat_inc(type);
4793 mtype_stat_dec(MT_FREE);
4794 }
4795 return m;
4796 }
4797
4798 /* m_mclget() add an mbuf cluster to a normal mbuf */
4799 struct mbuf *
4800 m_mclget(struct mbuf *m, int wait)
4801 {
4802 struct ext_ref *rfa = NULL;
4803
4804 #if CONFIG_MBUF_MCACHE
4805 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
4806 return m;
4807 }
4808 #else
4809 if ((rfa = mz_ref_alloc(wait)) == NULL) {
4810 return m;
4811 }
4812 #endif /* CONFIG_MBUF_MCACHE */
4813 m->m_ext.ext_buf = m_mclalloc(wait);
4814 if (m->m_ext.ext_buf != NULL) {
4815 MBUF_CL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
4816 } else {
4817 #if CONFIG_MBUF_MCACHE
4818 mcache_free(ref_cache, rfa);
4819 #else
4820 mz_ref_free(rfa);
4821 #endif /* CONFIG_MBUF_MCACHE */
4822 }
4823
4824 return m;
4825 }
4826
4827 /* Allocate an mbuf cluster */
4828 caddr_t
4829 m_mclalloc(int wait)
4830 {
4831 #if CONFIG_MBUF_MCACHE
4832 int mcflags = MSLEEPF(wait);
4833
4834 /* Is this due to a non-blocking retry? If so, then try harder */
4835 if (mcflags & MCR_NOSLEEP) {
4836 mcflags |= MCR_TRYHARD;
4837 }
4838
4839 return mcache_alloc(m_cache(MC_CL), mcflags);
4840 #else
4841 return mz_cl_alloc(ZONE_ID_CLUSTER_2K, wait);
4842 #endif /* CONFIG_MBUF_MCACHE */
4843 }
4844
4845 /* Free an mbuf cluster */
4846 void
4847 m_mclfree(caddr_t p)
4848 {
4849 #if CONFIG_MBUF_MCACHE
4850 mcache_free(m_cache(MC_CL), p);
4851 #else
4852 mz_cl_free(ZONE_ID_CLUSTER_2K, p);
4853 #endif /* CONFIG_MBUF_MCACHE */
4854 }
4855
4856 /*
4857 * mcl_hasreference() checks if a cluster of an mbuf is referenced by
4858 * another mbuf; see comments in m_incref() regarding EXTF_READONLY.
4859 */
4860 int
4861 m_mclhasreference(struct mbuf *m)
4862 {
4863 if (!(m->m_flags & M_EXT)) {
4864 return 0;
4865 }
4866
4867 ASSERT(m_get_rfa(m) != NULL);
4868
4869 return (MEXT_FLAGS(m) & EXTF_READONLY) ? 1 : 0;
4870 }
4871
4872 __private_extern__ caddr_t
4873 m_bigalloc(int wait)
4874 {
4875 #if CONFIG_MBUF_MCACHE
4876 int mcflags = MSLEEPF(wait);
4877
4878 /* Is this due to a non-blocking retry? If so, then try harder */
4879 if (mcflags & MCR_NOSLEEP) {
4880 mcflags |= MCR_TRYHARD;
4881 }
4882
4883 return mcache_alloc(m_cache(MC_BIGCL), mcflags);
4884 #else
4885 return mz_cl_alloc(ZONE_ID_CLUSTER_4K, wait);
4886 #endif /* CONFIG_MBUF_MCACHE */
4887 }
4888
4889 __private_extern__ void
4890 m_bigfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
4891 {
4892 #if CONFIG_MBUF_MCACHE
4893 mcache_free(m_cache(MC_BIGCL), p);
4894 #else
4895 mz_cl_free(ZONE_ID_CLUSTER_4K, p);
4896 #endif /* CONFIG_MBUF_MCACHE */
4897 }
4898
4899 /* m_mbigget() add an 4KB mbuf cluster to a normal mbuf */
4900 __private_extern__ struct mbuf *
4901 m_mbigget(struct mbuf *m, int wait)
4902 {
4903 struct ext_ref *rfa = NULL;
4904
4905 #if CONFIG_MBUF_MCACHE
4906 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
4907 return m;
4908 }
4909 #else
4910 if ((rfa = mz_ref_alloc(wait)) == NULL) {
4911 return m;
4912 }
4913 #endif /* CONFIG_MBUF_MCACHE */
4914 m->m_ext.ext_buf = m_bigalloc(wait);
4915 if (m->m_ext.ext_buf != NULL) {
4916 MBUF_BIGCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
4917 } else {
4918 #if CONFIG_MBUF_MCACHE
4919 mcache_free(ref_cache, rfa);
4920 #else
4921 mz_ref_free(rfa);
4922 #endif /* CONFIG_MBUF_MCACHE */
4923 }
4924 return m;
4925 }
4926
4927 __private_extern__ caddr_t
4928 m_16kalloc(int wait)
4929 {
4930 #if CONFIG_MBUF_MCACHE
4931 int mcflags = MSLEEPF(wait);
4932
4933 /* Is this due to a non-blocking retry? If so, then try harder */
4934 if (mcflags & MCR_NOSLEEP) {
4935 mcflags |= MCR_TRYHARD;
4936 }
4937
4938 return mcache_alloc(m_cache(MC_16KCL), mcflags);
4939 #else
4940 return mz_cl_alloc(ZONE_ID_CLUSTER_16K, wait);
4941 #endif /* CONFIG_MBUF_MCACHE */
4942 }
4943
4944 __private_extern__ void
4945 m_16kfree(caddr_t p, __unused u_int size, __unused caddr_t arg)
4946 {
4947 #if CONFIG_MBUF_MCACHE
4948 mcache_free(m_cache(MC_16KCL), p);
4949 #else
4950 mz_cl_free(ZONE_ID_CLUSTER_16K, p);
4951 #endif /* CONFIG_MBUF_MCACHE */
4952 }
4953
4954 /* m_m16kget() add a 16KB mbuf cluster to a normal mbuf */
4955 __private_extern__ struct mbuf *
4956 m_m16kget(struct mbuf *m, int wait)
4957 {
4958 struct ext_ref *rfa = NULL;
4959
4960 #if CONFIG_MBUF_MCACHE
4961 if ((rfa = mcache_alloc(ref_cache, MSLEEPF(wait))) == NULL) {
4962 return m;
4963 }
4964 #else
4965 if ((rfa = mz_ref_alloc(wait)) == NULL) {
4966 return m;
4967 }
4968 #endif /* CONFIG_MBUF_MCACHE */
4969 m->m_ext.ext_buf = m_16kalloc(wait);
4970 if (m->m_ext.ext_buf != NULL) {
4971 MBUF_16KCL_INIT(m, m->m_ext.ext_buf, rfa, 1, 0);
4972 } else {
4973 #if CONFIG_MBUF_MCACHE
4974 mcache_free(ref_cache, rfa);
4975 #else
4976 mz_ref_free(rfa);
4977 #endif /* CONFIG_MBUF_MCACHE */
4978 }
4979
4980 return m;
4981 }
4982
4983 /*
4984 * "Move" mbuf pkthdr from "from" to "to".
4985 * "from" must have M_PKTHDR set, and "to" must be empty.
4986 */
4987 void
4988 m_copy_pkthdr(struct mbuf *to, struct mbuf *from)
4989 {
4990 VERIFY(from->m_flags & M_PKTHDR);
4991
4992 /* Check for scratch area overflow */
4993 m_redzone_verify(from);
4994
4995 if (to->m_flags & M_PKTHDR) {
4996 /* Check for scratch area overflow */
4997 m_redzone_verify(to);
4998 /* We will be taking over the tags of 'to' */
4999 m_tag_delete_chain(to);
5000 }
5001 to->m_pkthdr = from->m_pkthdr; /* especially tags */
5002 m_classifier_init(from, 0); /* purge classifier info */
5003 m_tag_init(from, 1); /* purge all tags from src */
5004 m_scratch_init(from); /* clear src scratch area */
5005 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
5006 if ((to->m_flags & M_EXT) == 0) {
5007 to->m_data = (uintptr_t)to->m_pktdat;
5008 }
5009 m_redzone_init(to); /* setup red zone on dst */
5010 }
5011
5012 /*
5013 * Duplicate "from"'s mbuf pkthdr in "to".
5014 * "from" must have M_PKTHDR set, and "to" must be empty.
5015 * In particular, this does a deep copy of the packet tags.
5016 */
5017 int
5018 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
5019 {
5020 VERIFY(from->m_flags & M_PKTHDR);
5021
5022 /* Check for scratch area overflow */
5023 m_redzone_verify(from);
5024
5025 if (to->m_flags & M_PKTHDR) {
5026 /* Check for scratch area overflow */
5027 m_redzone_verify(to);
5028 /* We will be taking over the tags of 'to' */
5029 m_tag_delete_chain(to);
5030 }
5031 to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
5032 if ((to->m_flags & M_EXT) == 0) {
5033 to->m_data = (uintptr_t)to->m_pktdat;
5034 }
5035 to->m_pkthdr = from->m_pkthdr;
5036 /* clear TX completion flag so the callback is not called in the copy */
5037 to->m_pkthdr.pkt_flags &= ~PKTF_TX_COMPL_TS_REQ;
5038 m_redzone_init(to); /* setup red zone on dst */
5039 m_tag_init(to, 0); /* preserve dst static tags */
5040 return m_tag_copy_chain(to, from, how);
5041 }
5042
5043 void
5044 m_copy_pftag(struct mbuf *to, struct mbuf *from)
5045 {
5046 memcpy(m_pftag(to), m_pftag(from), sizeof(struct pf_mtag));
5047 #if PF_ECN
5048 m_pftag(to)->pftag_hdr = NULL;
5049 m_pftag(to)->pftag_flags &= ~(PF_TAG_HDR_INET | PF_TAG_HDR_INET6);
5050 #endif /* PF_ECN */
5051 }
5052
5053 void
5054 m_copy_necptag(struct mbuf *to, struct mbuf *from)
5055 {
5056 memcpy(m_necptag(to), m_necptag(from), sizeof(struct necp_mtag_));
5057 }
5058
5059 void
5060 m_classifier_init(struct mbuf *m, uint32_t pktf_mask)
5061 {
5062 VERIFY(m->m_flags & M_PKTHDR);
5063
5064 m->m_pkthdr.pkt_proto = 0;
5065 m->m_pkthdr.pkt_flowsrc = 0;
5066 m->m_pkthdr.pkt_flowid = 0;
5067 m->m_pkthdr.pkt_ext_flags = 0;
5068 m->m_pkthdr.pkt_flags &= pktf_mask; /* caller-defined mask */
5069 /* preserve service class and interface info for loopback packets */
5070 if (!(m->m_pkthdr.pkt_flags & PKTF_LOOP)) {
5071 (void) m_set_service_class(m, MBUF_SC_BE);
5072 }
5073 if (!(m->m_pkthdr.pkt_flags & PKTF_IFAINFO)) {
5074 m->m_pkthdr.pkt_ifainfo = 0;
5075 }
5076 /*
5077 * Preserve timestamp if requested
5078 */
5079 if (!(m->m_pkthdr.pkt_flags & PKTF_TS_VALID)) {
5080 m->m_pkthdr.pkt_timestamp = 0;
5081 }
5082 }
5083
5084 void
5085 m_copy_classifier(struct mbuf *to, struct mbuf *from)
5086 {
5087 VERIFY(to->m_flags & M_PKTHDR);
5088 VERIFY(from->m_flags & M_PKTHDR);
5089
5090 to->m_pkthdr.pkt_proto = from->m_pkthdr.pkt_proto;
5091 to->m_pkthdr.pkt_flowsrc = from->m_pkthdr.pkt_flowsrc;
5092 to->m_pkthdr.pkt_flowid = from->m_pkthdr.pkt_flowid;
5093 to->m_pkthdr.pkt_mpriv_srcid = from->m_pkthdr.pkt_mpriv_srcid;
5094 to->m_pkthdr.pkt_flags = from->m_pkthdr.pkt_flags;
5095 to->m_pkthdr.pkt_ext_flags = from->m_pkthdr.pkt_ext_flags;
5096 (void) m_set_service_class(to, from->m_pkthdr.pkt_svc);
5097 to->m_pkthdr.pkt_ifainfo = from->m_pkthdr.pkt_ifainfo;
5098 }
5099
5100 /*
5101 * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
5102 * if wantall is not set, return whatever number were available. Set up the
5103 * first num_with_pkthdrs with mbuf hdrs configured as packet headers; these
5104 * are chained on the m_nextpkt field. Any packets requested beyond this
5105 * are chained onto the last packet header's m_next field. The size of
5106 * the cluster is controlled by the parameter bufsize.
5107 */
5108 __private_extern__ struct mbuf *
5109 m_getpackets_internal(unsigned int *num_needed, int num_with_pkthdrs,
5110 int wait, int wantall, size_t bufsize)
5111 {
5112 struct mbuf *m = NULL;
5113 struct mbuf **np, *top;
5114 unsigned int pnum, needed = *num_needed;
5115 #if CONFIG_MBUF_MCACHE
5116 mcache_obj_t *mp_list = NULL;
5117 int mcflags = MSLEEPF(wait);
5118 mcache_t *cp;
5119 #else
5120 zstack_t mp_list = {};
5121 mbuf_class_t class = MC_MBUF_CL;
5122 #endif /* CONFIG_MBUF_MCACHE */
5123 u_int16_t flag;
5124 struct ext_ref *rfa;
5125 void *cl;
5126
5127 ASSERT(bufsize == m_maxsize(MC_CL) ||
5128 bufsize == m_maxsize(MC_BIGCL) ||
5129 bufsize == m_maxsize(MC_16KCL));
5130
5131 /*
5132 * Caller must first check for njcl because this
5133 * routine is internal and not exposed/used via KPI.
5134 */
5135 VERIFY(bufsize != m_maxsize(MC_16KCL) || njcl > 0);
5136
5137 top = NULL;
5138 np = ⊤
5139 pnum = 0;
5140
5141 /*
5142 * The caller doesn't want all the requested buffers; only some.
5143 * Try hard to get what we can, but don't block. This effectively
5144 * overrides MCR_SLEEP, since this thread will not go to sleep
5145 * if we can't get all the buffers.
5146 */
5147 #if CONFIG_MBUF_MCACHE
5148 if (!wantall || (mcflags & MCR_NOSLEEP)) {
5149 mcflags |= MCR_TRYHARD;
5150 }
5151
5152 /* Allocate the composite mbuf + cluster elements from the cache */
5153 if (bufsize == m_maxsize(MC_CL)) {
5154 cp = m_cache(MC_MBUF_CL);
5155 } else if (bufsize == m_maxsize(MC_BIGCL)) {
5156 cp = m_cache(MC_MBUF_BIGCL);
5157 } else {
5158 cp = m_cache(MC_MBUF_16KCL);
5159 }
5160 needed = mcache_alloc_ext(cp, &mp_list, needed, mcflags);
5161 #else
5162 if (!wantall || (wait & Z_NOWAIT)) {
5163 wait &= ~Z_NOWAIT;
5164 wait |= Z_NOPAGEWAIT;
5165 }
5166
5167 /* Allocate the composite mbuf + cluster elements from the cache */
5168 if (bufsize == m_maxsize(MC_CL)) {
5169 class = MC_MBUF_CL;
5170 } else if (bufsize == m_maxsize(MC_BIGCL)) {
5171 class = MC_MBUF_BIGCL;
5172 } else {
5173 class = MC_MBUF_16KCL;
5174 }
5175 mp_list = mz_composite_alloc_n(class, needed, wait);
5176 needed = zstack_count(mp_list);
5177 #endif /* CONFIG_MBUF_MCACHE */
5178
5179 for (pnum = 0; pnum < needed; pnum++) {
5180 #if CONFIG_MBUF_MCACHE
5181 m = (struct mbuf *)mp_list;
5182 mp_list = mp_list->obj_next;
5183 #else
5184 m = zstack_pop(&mp_list);
5185 #endif /* CONFIG_MBUF_MCACHE */
5186
5187 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
5188 cl = m->m_ext.ext_buf;
5189 rfa = m_get_rfa(m);
5190
5191 ASSERT(cl != NULL && rfa != NULL);
5192 VERIFY(MBUF_IS_COMPOSITE(m));
5193
5194 flag = MEXT_FLAGS(m);
5195
5196 MBUF_INIT(m, num_with_pkthdrs, MT_DATA);
5197 if (bufsize == m_maxsize(MC_16KCL)) {
5198 MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
5199 } else if (bufsize == m_maxsize(MC_BIGCL)) {
5200 MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
5201 } else {
5202 MBUF_CL_INIT(m, cl, rfa, 1, flag);
5203 }
5204
5205 if (num_with_pkthdrs > 0) {
5206 --num_with_pkthdrs;
5207 }
5208
5209 *np = m;
5210 if (num_with_pkthdrs > 0) {
5211 np = &m->m_nextpkt;
5212 } else {
5213 np = &m->m_next;
5214 }
5215 }
5216 #if CONFIG_MBUF_MCACHE
5217 ASSERT(pnum != *num_needed || mp_list == NULL);
5218 if (mp_list != NULL) {
5219 mcache_free_ext(cp, mp_list);
5220 }
5221 #else
5222 ASSERT(pnum != *num_needed || zstack_empty(mp_list));
5223 if (!zstack_empty(mp_list)) {
5224 mz_composite_free_n(class, mp_list);
5225 }
5226 #endif /* CONFIG_MBUF_MCACHE */
5227 if (pnum > 0) {
5228 mtype_stat_add(MT_DATA, pnum);
5229 mtype_stat_sub(MT_FREE, pnum);
5230 }
5231
5232 if (wantall && (pnum != *num_needed)) {
5233 if (top != NULL) {
5234 m_freem_list(top);
5235 }
5236 return NULL;
5237 }
5238
5239 if (pnum > *num_needed) {
5240 printf("%s: File a radar related to <rdar://10146739>. \
5241 needed = %u, pnum = %u, num_needed = %u \n",
5242 __func__, needed, pnum, *num_needed);
5243 }
5244 *num_needed = pnum;
5245
5246 return top;
5247 }
5248
5249 /*
5250 * Return list of mbuf linked by m_nextpkt. Try for numlist, and if
5251 * wantall is not set, return whatever number were available. The size of
5252 * each mbuf in the list is controlled by the parameter packetlen. Each
5253 * mbuf of the list may have a chain of mbufs linked by m_next. Each mbuf
5254 * in the chain is called a segment. If maxsegments is not null and the
5255 * value pointed to is not null, this specify the maximum number of segments
5256 * for a chain of mbufs. If maxsegments is zero or the value pointed to
5257 * is zero the caller does not have any restriction on the number of segments.
5258 * The actual number of segments of a mbuf chain is return in the value
5259 * pointed to by maxsegments.
5260 */
5261 __private_extern__ struct mbuf *
5262 m_allocpacket_internal(unsigned int *numlist, size_t packetlen,
5263 unsigned int *maxsegments, int wait, int wantall, size_t wantsize)
5264 {
5265 struct mbuf **np, *top, *first = NULL;
5266 size_t bufsize, r_bufsize;
5267 unsigned int num = 0;
5268 unsigned int nsegs = 0;
5269 unsigned int needed = 0, resid;
5270 #if CONFIG_MBUF_MCACHE
5271 int mcflags = MSLEEPF(wait);
5272 mcache_obj_t *mp_list = NULL, *rmp_list = NULL;
5273 mcache_t *cp = NULL, *rcp = NULL;
5274 #else
5275 zstack_t mp_list = {}, rmp_list = {};
5276 mbuf_class_t class = MC_MBUF, rclass = MC_MBUF_CL;
5277 #endif /* CONFIG_MBUF_MCACHE */
5278
5279 if (*numlist == 0) {
5280 os_log(OS_LOG_DEFAULT, "m_allocpacket_internal *numlist is 0");
5281 return NULL;
5282 }
5283
5284 top = NULL;
5285 np = ⊤
5286
5287 if (wantsize == 0) {
5288 if (packetlen <= MINCLSIZE) {
5289 bufsize = packetlen;
5290 } else if (packetlen > m_maxsize(MC_CL)) {
5291 /* Use 4KB if jumbo cluster pool isn't available */
5292 if (packetlen <= m_maxsize(MC_BIGCL) || njcl == 0) {
5293 bufsize = m_maxsize(MC_BIGCL);
5294 } else {
5295 bufsize = m_maxsize(MC_16KCL);
5296 }
5297 } else {
5298 bufsize = m_maxsize(MC_CL);
5299 }
5300 } else if (wantsize == m_maxsize(MC_CL) ||
5301 wantsize == m_maxsize(MC_BIGCL) ||
5302 (wantsize == m_maxsize(MC_16KCL) && njcl > 0)) {
5303 bufsize = wantsize;
5304 } else {
5305 *numlist = 0;
5306 os_log(OS_LOG_DEFAULT, "m_allocpacket_internal wantsize unsupported");
5307 return NULL;
5308 }
5309
5310 if (bufsize <= MHLEN) {
5311 nsegs = 1;
5312 } else if (bufsize <= MINCLSIZE) {
5313 if (maxsegments != NULL && *maxsegments == 1) {
5314 bufsize = m_maxsize(MC_CL);
5315 nsegs = 1;
5316 } else {
5317 nsegs = 2;
5318 }
5319 } else if (bufsize == m_maxsize(MC_16KCL)) {
5320 VERIFY(njcl > 0);
5321 nsegs = ((packetlen - 1) >> M16KCLSHIFT) + 1;
5322 } else if (bufsize == m_maxsize(MC_BIGCL)) {
5323 nsegs = ((packetlen - 1) >> MBIGCLSHIFT) + 1;
5324 } else {
5325 nsegs = ((packetlen - 1) >> MCLSHIFT) + 1;
5326 }
5327 if (maxsegments != NULL) {
5328 if (*maxsegments && nsegs > *maxsegments) {
5329 *maxsegments = nsegs;
5330 *numlist = 0;
5331 os_log(OS_LOG_DEFAULT, "m_allocpacket_internal nsegs > *maxsegments");
5332 return NULL;
5333 }
5334 *maxsegments = nsegs;
5335 }
5336
5337 /*
5338 * The caller doesn't want all the requested buffers; only some.
5339 * Try hard to get what we can, but don't block. This effectively
5340 * overrides MCR_SLEEP, since this thread will not go to sleep
5341 * if we can't get all the buffers.
5342 */
5343 #if CONFIG_MBUF_MCACHE
5344 if (!wantall || (mcflags & MCR_NOSLEEP)) {
5345 mcflags |= MCR_TRYHARD;
5346 }
5347 #else
5348 if (!wantall || (wait & Z_NOWAIT)) {
5349 wait &= ~Z_NOWAIT;
5350 wait |= Z_NOPAGEWAIT;
5351 }
5352 #endif /* !CONFIG_MBUF_MCACHE */
5353
5354 /*
5355 * Simple case where all elements in the lists/chains are mbufs.
5356 * Unless bufsize is greater than MHLEN, each segment chain is made
5357 * up of exactly 1 mbuf. Otherwise, each segment chain is made up
5358 * of 2 mbufs; the second one is used for the residual data, i.e.
5359 * the remaining data that cannot fit into the first mbuf.
5360 */
5361 if (bufsize <= MINCLSIZE) {
5362 /* Allocate the elements in one shot from the mbuf cache */
5363 ASSERT(bufsize <= MHLEN || nsegs == 2);
5364 #if CONFIG_MBUF_MCACHE
5365 cp = m_cache(MC_MBUF);
5366 needed = mcache_alloc_ext(cp, &mp_list,
5367 (*numlist) * nsegs, mcflags);
5368 #else
5369 class = MC_MBUF;
5370 mp_list = mz_alloc_n((*numlist) * nsegs, wait);
5371 needed = zstack_count(mp_list);
5372 #endif /* CONFIG_MBUF_MCACHE */
5373
5374 /*
5375 * The number of elements must be even if we are to use an
5376 * mbuf (instead of a cluster) to store the residual data.
5377 * If we couldn't allocate the requested number of mbufs,
5378 * trim the number down (if it's odd) in order to avoid
5379 * creating a partial segment chain.
5380 */
5381 if (bufsize > MHLEN && (needed & 0x1)) {
5382 needed--;
5383 }
5384
5385 while (num < needed) {
5386 struct mbuf *m = NULL;
5387
5388 #if CONFIG_MBUF_MCACHE
5389 m = (struct mbuf *)mp_list;
5390 mp_list = mp_list->obj_next;
5391 #else
5392 m = zstack_pop(&mp_list);
5393 #endif /* CONFIG_MBUF_MCACHE */
5394 ASSERT(m != NULL);
5395
5396 MBUF_INIT(m, 1, MT_DATA);
5397 num++;
5398 if (bufsize > MHLEN) {
5399 /* A second mbuf for this segment chain */
5400 #if CONFIG_MBUF_MCACHE
5401 m->m_next = (struct mbuf *)mp_list;
5402 mp_list = mp_list->obj_next;
5403 #else
5404 m->m_next = zstack_pop(&mp_list);
5405 #endif /* CONFIG_MBUF_MCACHE */
5406
5407 ASSERT(m->m_next != NULL);
5408
5409 MBUF_INIT(m->m_next, 0, MT_DATA);
5410 num++;
5411 }
5412 *np = m;
5413 np = &m->m_nextpkt;
5414 }
5415 #if CONFIG_MBUF_MCACHE
5416 ASSERT(num != *numlist || mp_list == NULL);
5417 #else
5418 ASSERT(num != *numlist || zstack_empty(mp_list));
5419 #endif /* CONFIG_MBUF_MCACHE */
5420
5421 if (num > 0) {
5422 mtype_stat_add(MT_DATA, num);
5423 mtype_stat_sub(MT_FREE, num);
5424 }
5425 num /= nsegs;
5426
5427 /* We've got them all; return to caller */
5428 if (num == *numlist) {
5429 return top;
5430 }
5431
5432 goto fail;
5433 }
5434
5435 /*
5436 * Complex cases where elements are made up of one or more composite
5437 * mbufs + cluster, depending on packetlen. Each N-segment chain can
5438 * be illustrated as follows:
5439 *
5440 * [mbuf + cluster 1] [mbuf + cluster 2] ... [mbuf + cluster N]
5441 *
5442 * Every composite mbuf + cluster element comes from the intermediate
5443 * cache (either MC_MBUF_CL or MC_MBUF_BIGCL). For space efficiency,
5444 * the last composite element will come from the MC_MBUF_CL cache,
5445 * unless the residual data is larger than 2KB where we use the
5446 * big cluster composite cache (MC_MBUF_BIGCL) instead. Residual
5447 * data is defined as extra data beyond the first element that cannot
5448 * fit into the previous element, i.e. there is no residual data if
5449 * the chain only has 1 segment.
5450 */
5451 r_bufsize = bufsize;
5452 resid = packetlen > bufsize ? packetlen % bufsize : 0;
5453 if (resid > 0) {
5454 /* There is residual data; figure out the cluster size */
5455 if (wantsize == 0 && packetlen > MINCLSIZE) {
5456 /*
5457 * Caller didn't request that all of the segments
5458 * in the chain use the same cluster size; use the
5459 * smaller of the cluster sizes.
5460 */
5461 if (njcl > 0 && resid > m_maxsize(MC_BIGCL)) {
5462 r_bufsize = m_maxsize(MC_16KCL);
5463 } else if (resid > m_maxsize(MC_CL)) {
5464 r_bufsize = m_maxsize(MC_BIGCL);
5465 } else {
5466 r_bufsize = m_maxsize(MC_CL);
5467 }
5468 } else {
5469 /* Use the same cluster size as the other segments */
5470 resid = 0;
5471 }
5472 }
5473
5474 needed = *numlist;
5475 if (resid > 0) {
5476 /*
5477 * Attempt to allocate composite mbuf + cluster elements for
5478 * the residual data in each chain; record the number of such
5479 * elements that can be allocated so that we know how many
5480 * segment chains we can afford to create.
5481 */
5482 #if CONFIG_MBUF_MCACHE
5483 if (r_bufsize <= m_maxsize(MC_CL)) {
5484 rcp = m_cache(MC_MBUF_CL);
5485 } else if (r_bufsize <= m_maxsize(MC_BIGCL)) {
5486 rcp = m_cache(MC_MBUF_BIGCL);
5487 } else {
5488 rcp = m_cache(MC_MBUF_16KCL);
5489 }
5490 needed = mcache_alloc_ext(rcp, &rmp_list, *numlist, mcflags);
5491 #else
5492 if (r_bufsize <= m_maxsize(MC_CL)) {
5493 rclass = MC_MBUF_CL;
5494 } else if (r_bufsize <= m_maxsize(MC_BIGCL)) {
5495 rclass = MC_MBUF_BIGCL;
5496 } else {
5497 rclass = MC_MBUF_16KCL;
5498 }
5499 rmp_list = mz_composite_alloc_n(rclass, *numlist, wait);
5500 needed = zstack_count(rmp_list);
5501 #endif /* CONFIG_MBUF_MCACHE */
5502 if (needed == 0) {
5503 goto fail;
5504 }
5505
5506 /* This is temporarily reduced for calculation */
5507 ASSERT(nsegs > 1);
5508 nsegs--;
5509 }
5510
5511 /*
5512 * Attempt to allocate the rest of the composite mbuf + cluster
5513 * elements for the number of segment chains that we need.
5514 */
5515 #if CONFIG_MBUF_MCACHE
5516 if (bufsize <= m_maxsize(MC_CL)) {
5517 cp = m_cache(MC_MBUF_CL);
5518 } else if (bufsize <= m_maxsize(MC_BIGCL)) {
5519 cp = m_cache(MC_MBUF_BIGCL);
5520 } else {
5521 cp = m_cache(MC_MBUF_16KCL);
5522 }
5523 needed = mcache_alloc_ext(cp, &mp_list, needed * nsegs, mcflags);
5524 #else
5525 if (bufsize <= m_maxsize(MC_CL)) {
5526 class = MC_MBUF_CL;
5527 } else if (bufsize <= m_maxsize(MC_BIGCL)) {
5528 class = MC_MBUF_BIGCL;
5529 } else {
5530 class = MC_MBUF_16KCL;
5531 }
5532 mp_list = mz_composite_alloc_n(class, needed * nsegs, wait);
5533 needed = zstack_count(mp_list);
5534 #endif /* CONFIG_MBUF_MCACHE */
5535
5536 /* Round it down to avoid creating a partial segment chain */
5537 needed = (needed / nsegs) * nsegs;
5538 if (needed == 0) {
5539 goto fail;
5540 }
5541
5542 if (resid > 0) {
5543 /*
5544 * We're about to construct the chain(s); take into account
5545 * the number of segments we have created above to hold the
5546 * residual data for each chain, as well as restore the
5547 * original count of segments per chain.
5548 */
5549 ASSERT(nsegs > 0);
5550 needed += needed / nsegs;
5551 nsegs++;
5552 }
5553
5554 for (;;) {
5555 struct mbuf *m = NULL;
5556 u_int16_t flag;
5557 struct ext_ref *rfa;
5558 void *cl;
5559 int pkthdr;
5560 m_ext_free_func_t m_free_func;
5561
5562 ++num;
5563
5564 if (nsegs == 1 || (num % nsegs) != 0 || resid == 0) {
5565 #if CONFIG_MBUF_MCACHE
5566 m = (struct mbuf *)mp_list;
5567 mp_list = mp_list->obj_next;
5568 #else
5569 m = zstack_pop(&mp_list);
5570 #endif /* CONFIG_MBUF_MCACHE */
5571 } else {
5572 #if CONFIG_MBUF_MCACHE
5573 m = (struct mbuf *)rmp_list;
5574 rmp_list = rmp_list->obj_next;
5575 #else
5576 m = zstack_pop(&rmp_list);
5577 #endif /* CONFIG_MBUF_MCACHE */
5578 }
5579 m_free_func = m_get_ext_free(m);
5580 ASSERT(m != NULL);
5581 VERIFY(m->m_type == MT_FREE && m->m_flags == M_EXT);
5582 VERIFY(m_free_func == NULL || m_free_func == m_bigfree ||
5583 m_free_func == m_16kfree);
5584
5585 cl = m->m_ext.ext_buf;
5586 rfa = m_get_rfa(m);
5587
5588 ASSERT(cl != NULL && rfa != NULL);
5589 VERIFY(MBUF_IS_COMPOSITE(m));
5590
5591 flag = MEXT_FLAGS(m);
5592
5593 pkthdr = (nsegs == 1 || (num % nsegs) == 1);
5594 if (pkthdr) {
5595 first = m;
5596 }
5597 MBUF_INIT(m, pkthdr, MT_DATA);
5598 if (m_free_func == m_16kfree) {
5599 MBUF_16KCL_INIT(m, cl, rfa, 1, flag);
5600 } else if (m_free_func == m_bigfree) {
5601 MBUF_BIGCL_INIT(m, cl, rfa, 1, flag);
5602 } else {
5603 MBUF_CL_INIT(m, cl, rfa, 1, flag);
5604 }
5605
5606 *np = m;
5607 if ((num % nsegs) == 0) {
5608 np = &first->m_nextpkt;
5609 } else {
5610 np = &m->m_next;
5611 }
5612
5613 if (num == needed) {
5614 break;
5615 }
5616 }
5617
5618 if (num > 0) {
5619 mtype_stat_add(MT_DATA, num);
5620 mtype_stat_sub(MT_FREE, num);
5621 }
5622
5623 num /= nsegs;
5624
5625 /* We've got them all; return to caller */
5626 if (num == *numlist) {
5627 #if CONFIG_MBUF_MCACHE
5628 ASSERT(mp_list == NULL && rmp_list == NULL);
5629 #else
5630 ASSERT(zstack_empty(mp_list) && zstack_empty(rmp_list));
5631 #endif /* CONFIG_MBUF_MCACHE */
5632 return top;
5633 }
5634
5635 fail:
5636 /* Free up what's left of the above */
5637 #if CONFIG_MBUF_MCACHE
5638 if (mp_list != NULL) {
5639 mcache_free_ext(cp, mp_list);
5640 }
5641 if (rmp_list != NULL) {
5642 mcache_free_ext(rcp, rmp_list);
5643 }
5644 #else
5645 if (!zstack_empty(mp_list)) {
5646 if (class == MC_MBUF) {
5647 /* No need to elide, these mbufs came from the cache. */
5648 mz_free_n(mp_list);
5649 } else {
5650 mz_composite_free_n(class, mp_list);
5651 }
5652 }
5653 if (!zstack_empty(rmp_list)) {
5654 mz_composite_free_n(rclass, rmp_list);
5655 }
5656 #endif /* CONFIG_MBUF_MCACHE */
5657 if (wantall && top != NULL) {
5658 m_freem_list(top);
5659 *numlist = 0;
5660 return NULL;
5661 }
5662 *numlist = num;
5663 return top;
5664 }
5665
5666 /*
5667 * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
5668 * packets on receive ring.
5669 */
5670 __private_extern__ struct mbuf *
5671 m_getpacket_how(int wait)
5672 {
5673 unsigned int num_needed = 1;
5674
5675 return m_getpackets_internal(&num_needed, 1, wait, 1,
5676 m_maxsize(MC_CL));
5677 }
5678
5679 /*
5680 * Best effort to get a mbuf cluster + pkthdr. Used by drivers to allocated
5681 * packets on receive ring.
5682 */
5683 struct mbuf *
5684 m_getpacket(void)
5685 {
5686 unsigned int num_needed = 1;
5687
5688 return m_getpackets_internal(&num_needed, 1, M_WAIT, 1,
5689 m_maxsize(MC_CL));
5690 }
5691
5692 /*
5693 * Return a list of mbuf hdrs that point to clusters. Try for num_needed;
5694 * if this can't be met, return whatever number were available. Set up the
5695 * first num_with_pkthdrs with mbuf hdrs configured as packet headers. These
5696 * are chained on the m_nextpkt field. Any packets requested beyond this are
5697 * chained onto the last packet header's m_next field.
5698 */
5699 struct mbuf *
5700 m_getpackets(int num_needed, int num_with_pkthdrs, int how)
5701 {
5702 unsigned int n = num_needed;
5703
5704 return m_getpackets_internal(&n, num_with_pkthdrs, how, 0,
5705 m_maxsize(MC_CL));
5706 }
5707
5708 /*
5709 * Return a list of mbuf hdrs set up as packet hdrs chained together
5710 * on the m_nextpkt field
5711 */
5712 struct mbuf *
5713 m_getpackethdrs(int num_needed, int how)
5714 {
5715 struct mbuf *m;
5716 struct mbuf **np, *top;
5717
5718 top = NULL;
5719 np = ⊤
5720
5721 while (num_needed--) {
5722 m = _M_RETRYHDR(how, MT_DATA);
5723 if (m == NULL) {
5724 break;
5725 }
5726
5727 *np = m;
5728 np = &m->m_nextpkt;
5729 }
5730
5731 return top;
5732 }
5733
5734 /*
5735 * Free an mbuf list (m_nextpkt) while following m_next. Returns the count
5736 * for mbufs packets freed. Used by the drivers.
5737 */
5738 int
5739 m_freem_list(struct mbuf *m)
5740 {
5741 struct mbuf *nextpkt;
5742 #if CONFIG_MBUF_MCACHE
5743 mcache_obj_t *mp_list = NULL;
5744 mcache_obj_t *mcl_list = NULL;
5745 mcache_obj_t *mbc_list = NULL;
5746 mcache_obj_t *m16k_list = NULL;
5747 mcache_obj_t *m_mcl_list = NULL;
5748 mcache_obj_t *m_mbc_list = NULL;
5749 mcache_obj_t *m_m16k_list = NULL;
5750 mcache_obj_t *ref_list = NULL;
5751 #else
5752 zstack_t mp_list = {}, mcl_list = {}, mbc_list = {},
5753 m16k_list = {}, m_mcl_list = {},
5754 m_mbc_list = {}, m_m16k_list = {}, ref_list = {};
5755 #endif /* CONFIG_MBUF_MCACHE */
5756 int pktcount = 0;
5757 int mt_free = 0, mt_data = 0, mt_header = 0, mt_soname = 0, mt_tag = 0;
5758
5759 while (m != NULL) {
5760 pktcount++;
5761
5762 nextpkt = m->m_nextpkt;
5763 m->m_nextpkt = NULL;
5764
5765 while (m != NULL) {
5766 struct mbuf *next = m->m_next;
5767 #if CONFIG_MBUF_MCACHE
5768 mcache_obj_t *o, *rfa;
5769 #else
5770 void *cl = NULL;
5771 #endif /* CONFIG_MBUF_MCACHE */
5772 if (m->m_type == MT_FREE) {
5773 panic("m_free: freeing an already freed mbuf");
5774 }
5775
5776 if (m->m_flags & M_PKTHDR) {
5777 /* Check for scratch area overflow */
5778 m_redzone_verify(m);
5779 /* Free the aux data and tags if there is any */
5780 m_tag_delete_chain(m);
5781 m_do_tx_compl_callback(m, NULL);
5782 }
5783
5784 if (!(m->m_flags & M_EXT)) {
5785 mt_free++;
5786 goto simple_free;
5787 }
5788
5789 if (MBUF_IS_PAIRED(m) && m_free_paired(m)) {
5790 m = next;
5791 continue;
5792 }
5793
5794 mt_free++;
5795
5796 #if CONFIG_MBUF_MCACHE
5797 o = (mcache_obj_t *)(void *)m->m_ext.ext_buf;
5798 #else
5799 cl = m->m_ext.ext_buf;
5800 #endif /* CONFIG_MBUF_MCACHE */
5801 /*
5802 * Make sure that we don't touch any ext_ref
5803 * member after we decrement the reference count
5804 * since that may lead to use-after-free
5805 * when we do not hold the last reference.
5806 */
5807 const bool composite = !!(MEXT_FLAGS(m) & EXTF_COMPOSITE);
5808 const m_ext_free_func_t m_free_func = m_get_ext_free(m);
5809 const uint16_t minref = MEXT_MINREF(m);
5810 const uint16_t refcnt = m_decref(m);
5811 if (refcnt == minref && !composite) {
5812 #if CONFIG_MBUF_MCACHE
5813 if (m_free_func == NULL) {
5814 o->obj_next = mcl_list;
5815 mcl_list = o;
5816 } else if (m_free_func == m_bigfree) {
5817 o->obj_next = mbc_list;
5818 mbc_list = o;
5819 } else if (m_free_func == m_16kfree) {
5820 o->obj_next = m16k_list;
5821 m16k_list = o;
5822 } else {
5823 (*(m_free_func))((caddr_t)o,
5824 m->m_ext.ext_size,
5825 m_get_ext_arg(m));
5826 }
5827 rfa = (mcache_obj_t *)(void *)m_get_rfa(m);
5828 rfa->obj_next = ref_list;
5829 ref_list = rfa;
5830 #else
5831 if (m_free_func == NULL) {
5832 zstack_push(&mcl_list, cl);
5833 } else if (m_free_func == m_bigfree) {
5834 zstack_push(&mbc_list, cl);
5835 } else if (m_free_func == m_16kfree) {
5836 zstack_push(&m16k_list, cl);
5837 } else {
5838 (*(m_free_func))((caddr_t)cl,
5839 m->m_ext.ext_size,
5840 m_get_ext_arg(m));
5841 }
5842 zstack_push(&ref_list, m_get_rfa(m));
5843 #endif /* CONFIG_MBUF_MCACHE */
5844 m_set_ext(m, NULL, NULL, NULL);
5845 } else if (refcnt == minref && composite) {
5846 VERIFY(!(MEXT_FLAGS(m) & EXTF_PAIRED));
5847 /*
5848 * Amortize the costs of atomic operations
5849 * by doing them at the end, if possible.
5850 */
5851 if (m->m_type == MT_DATA) {
5852 mt_data++;
5853 } else if (m->m_type == MT_HEADER) {
5854 mt_header++;
5855 } else if (m->m_type == MT_SONAME) {
5856 mt_soname++;
5857 } else if (m->m_type == MT_TAG) {
5858 mt_tag++;
5859 } else {
5860 mtype_stat_dec(m->m_type);
5861 }
5862
5863 m->m_type = MT_FREE;
5864 m->m_flags = M_EXT;
5865 m->m_len = 0;
5866 m->m_next = m->m_nextpkt = NULL;
5867
5868 /*
5869 * MEXT_FLAGS is safe to access here
5870 * since we are now sure that we held
5871 * the last reference to ext_ref.
5872 */
5873 MEXT_FLAGS(m) &= ~EXTF_READONLY;
5874
5875 /* "Free" into the intermediate cache */
5876 #if CONFIG_MBUF_MCACHE
5877 o = (mcache_obj_t *)m;
5878 if (m_free_func == NULL) {
5879 o->obj_next = m_mcl_list;
5880 m_mcl_list = o;
5881 } else if (m_free_func == m_bigfree) {
5882 o->obj_next = m_mbc_list;
5883 m_mbc_list = o;
5884 } else {
5885 VERIFY(m_free_func == m_16kfree);
5886 o->obj_next = m_m16k_list;
5887 m_m16k_list = o;
5888 }
5889 #else
5890 if (m_free_func == NULL) {
5891 zstack_push(&m_mcl_list, m);
5892 } else if (m_free_func == m_bigfree) {
5893 zstack_push(&m_mbc_list, m);
5894 } else {
5895 VERIFY(m_free_func == m_16kfree);
5896 zstack_push(&m_m16k_list, m);
5897 }
5898 #endif /* CONFIG_MBUF_MCACHE */
5899 m = next;
5900 continue;
5901 }
5902 simple_free:
5903 /*
5904 * Amortize the costs of atomic operations
5905 * by doing them at the end, if possible.
5906 */
5907 if (m->m_type == MT_DATA) {
5908 mt_data++;
5909 } else if (m->m_type == MT_HEADER) {
5910 mt_header++;
5911 } else if (m->m_type == MT_SONAME) {
5912 mt_soname++;
5913 } else if (m->m_type == MT_TAG) {
5914 mt_tag++;
5915 } else if (m->m_type != MT_FREE) {
5916 mtype_stat_dec(m->m_type);
5917 }
5918
5919 m->m_type = MT_FREE;
5920 m->m_flags = m->m_len = 0;
5921 m->m_next = m->m_nextpkt = NULL;
5922
5923 #if CONFIG_MBUF_MCACHE
5924 ((mcache_obj_t *)m)->obj_next = mp_list;
5925 mp_list = (mcache_obj_t *)m;
5926 #else
5927 m_elide(m);
5928 zstack_push(&mp_list, m);
5929 #endif /* CONFIG_MBUF_MCACHE */
5930
5931 m = next;
5932 }
5933
5934 m = nextpkt;
5935 }
5936
5937 if (mt_free > 0) {
5938 mtype_stat_add(MT_FREE, mt_free);
5939 }
5940 if (mt_data > 0) {
5941 mtype_stat_sub(MT_DATA, mt_data);
5942 }
5943 if (mt_header > 0) {
5944 mtype_stat_sub(MT_HEADER, mt_header);
5945 }
5946 if (mt_soname > 0) {
5947 mtype_stat_sub(MT_SONAME, mt_soname);
5948 }
5949 if (mt_tag > 0) {
5950 mtype_stat_sub(MT_TAG, mt_tag);
5951 }
5952 #if CONFIG_MBUF_MCACHE
5953 if (mp_list != NULL) {
5954 mcache_free_ext(m_cache(MC_MBUF), mp_list);
5955 }
5956 if (mcl_list != NULL) {
5957 mcache_free_ext(m_cache(MC_CL), mcl_list);
5958 }
5959 if (mbc_list != NULL) {
5960 mcache_free_ext(m_cache(MC_BIGCL), mbc_list);
5961 }
5962 if (m16k_list != NULL) {
5963 mcache_free_ext(m_cache(MC_16KCL), m16k_list);
5964 }
5965 if (m_mcl_list != NULL) {
5966 mcache_free_ext(m_cache(MC_MBUF_CL), m_mcl_list);
5967 }
5968 if (m_mbc_list != NULL) {
5969 mcache_free_ext(m_cache(MC_MBUF_BIGCL), m_mbc_list);
5970 }
5971 if (m_m16k_list != NULL) {
5972 mcache_free_ext(m_cache(MC_MBUF_16KCL), m_m16k_list);
5973 }
5974 if (ref_list != NULL) {
5975 mcache_free_ext(ref_cache, ref_list);
5976 }
5977 #else
5978 if (!zstack_empty(mp_list)) {
5979 /* mbufs elided above. */
5980 mz_free_n(mp_list);
5981 }
5982 if (!zstack_empty(mcl_list)) {
5983 zfree_nozero_n(ZONE_ID_CLUSTER_2K, mcl_list);
5984 }
5985 if (!zstack_empty(mbc_list)) {
5986 zfree_nozero_n(ZONE_ID_CLUSTER_4K, mbc_list);
5987 }
5988 if (!zstack_empty(m16k_list)) {
5989 zfree_nozero_n(ZONE_ID_CLUSTER_16K, m16k_list);
5990 }
5991 if (!zstack_empty(m_mcl_list)) {
5992 mz_composite_free_n(MC_MBUF_CL, m_mcl_list);
5993 }
5994 if (!zstack_empty(m_mbc_list)) {
5995 mz_composite_free_n(MC_MBUF_BIGCL, m_mbc_list);
5996 }
5997 if (!zstack_empty(m_m16k_list)) {
5998 mz_composite_free_n(MC_MBUF_16KCL, m_m16k_list);
5999 }
6000 if (!zstack_empty(ref_list)) {
6001 zfree_nozero_n(ZONE_ID_MBUF_REF, ref_list);
6002 }
6003 #endif /* CONFIG_MBUF_MCACHE */
6004
6005 return pktcount;
6006 }
6007
6008 /*
6009 * Wrapper around m_freem_list which captures the packet that's going to be
6010 * dropped. If funcname is NULL, that means we do not want to store both
6011 * function name and line number, and only the drop reason will be saved.
6012 * Make sure to pass the direction flag (DROPTAP_FLAG_DIR_OUT,
6013 * DROPTAP_FLAG_DIR_IN), or the packet will not be captured.
6014 */
6015 void
6016 m_drop_list(mbuf_t m, struct ifnet *ifp, uint16_t flags, uint32_t reason, const char *funcname,
6017 uint16_t linenum)
6018 {
6019 struct mbuf *nextpkt;
6020
6021 if (m == NULL) {
6022 return;
6023 }
6024
6025 if (__probable(droptap_total_tap_count == 0)) {
6026 m_freem_list(m);
6027 return;
6028 }
6029
6030 if (flags & DROPTAP_FLAG_DIR_OUT) {
6031 while (m != NULL) {
6032 uint16_t tmp_flags = flags;
6033
6034 nextpkt = m->m_nextpkt;
6035 if (m->m_pkthdr.pkt_hdr == NULL) {
6036 tmp_flags |= DROPTAP_FLAG_L2_MISSING;
6037 }
6038 droptap_output_mbuf(m, reason, funcname, linenum, tmp_flags,
6039 ifp);
6040 m = nextpkt;
6041 }
6042 } else if (flags & DROPTAP_FLAG_DIR_IN) {
6043 while (m != NULL) {
6044 char *frame_header;
6045 uint16_t tmp_flags = flags;
6046
6047 nextpkt = m->m_nextpkt;
6048
6049 if ((flags & DROPTAP_FLAG_L2_MISSING) == 0 &&
6050 m->m_pkthdr.pkt_hdr != NULL) {
6051 frame_header = m->m_pkthdr.pkt_hdr;
6052 } else {
6053 frame_header = NULL;
6054 tmp_flags |= DROPTAP_FLAG_L2_MISSING;
6055 }
6056
6057 droptap_input_mbuf(m, reason, funcname, linenum, tmp_flags,
6058 m->m_pkthdr.rcvif, frame_header);
6059 m = nextpkt;
6060 }
6061 }
6062 m_freem_list(m);
6063 }
6064
6065 void
6066 m_freem(struct mbuf *m)
6067 {
6068 while (m != NULL) {
6069 m = m_free(m);
6070 }
6071 }
6072
6073 /*
6074 * Wrapper around m_freem which captures the packet that's going to be dropped.
6075 * If funcname is NULL, that means we do not want to store both function name
6076 * and line number, and only the drop reason will be saved. Make sure to pass the
6077 * direction flag (DROPTAP_FLAG_DIR_OUT, DROPTAP_FLAG_DIR_IN), or the packet will
6078 * not be captured.
6079 */
6080 void
6081 m_drop(mbuf_t m, uint16_t flags, uint32_t reason, const char *funcname,
6082 uint16_t linenum)
6083 {
6084 struct ifnet *ifp = NULL;
6085
6086 if (m == NULL) {
6087 return;
6088 }
6089
6090 if (__probable(droptap_total_tap_count == 0)) {
6091 m_freem(m);
6092 return;
6093 }
6094
6095 if (flags & DROPTAP_FLAG_DIR_OUT) {
6096 droptap_output_mbuf(m, reason, funcname, linenum, flags, ifp);
6097 } else if (flags & DROPTAP_FLAG_DIR_IN) {
6098 char *frame_header;
6099
6100 ifp = m->m_pkthdr.rcvif;
6101
6102 if ((flags & DROPTAP_FLAG_L2_MISSING) == 0 &&
6103 m->m_pkthdr.pkt_hdr != NULL) {
6104 frame_header = m->m_pkthdr.pkt_hdr;
6105 } else {
6106 frame_header = NULL;
6107 flags |= DROPTAP_FLAG_L2_MISSING;
6108 }
6109
6110 droptap_input_mbuf(m, reason, funcname, linenum, flags, ifp,
6111 frame_header);
6112 }
6113 m_freem(m);
6114 }
6115
6116 /*
6117 * Mbuffer utility routines.
6118 */
6119 /*
6120 * Set the m_data pointer of a newly allocated mbuf to place an object of the
6121 * specified size at the end of the mbuf, longword aligned.
6122 *
6123 * NB: Historically, we had M_ALIGN(), MH_ALIGN(), and MEXT_ALIGN() as
6124 * separate macros, each asserting that it was called at the proper moment.
6125 * This required callers to themselves test the storage type and call the
6126 * right one. Rather than require callers to be aware of those layout
6127 * decisions, we centralize here.
6128 */
6129 void
6130 m_align(struct mbuf *m, int len)
6131 {
6132 int adjust = 0;
6133
6134 /* At this point data must point to start */
6135 VERIFY(m->m_data == (uintptr_t)M_START(m));
6136 VERIFY(len >= 0);
6137 VERIFY(len <= M_SIZE(m));
6138 adjust = M_SIZE(m) - len;
6139 m->m_data += adjust & ~(sizeof(long) - 1);
6140 }
6141
6142 /*
6143 * Lesser-used path for M_PREPEND: allocate new mbuf to prepend to chain,
6144 * copy junk along. Does not adjust packet header length.
6145 */
6146 struct mbuf *
6147 m_prepend(struct mbuf *m, int len, int how)
6148 {
6149 struct mbuf *mn;
6150
6151 _MGET(mn, how, m->m_type);
6152 if (mn == NULL) {
6153 m_freem(m);
6154 return NULL;
6155 }
6156 if (m->m_flags & M_PKTHDR) {
6157 M_COPY_PKTHDR(mn, m);
6158 m->m_flags &= ~M_PKTHDR;
6159 }
6160 mn->m_next = m;
6161 m = mn;
6162 if (m->m_flags & M_PKTHDR) {
6163 VERIFY(len <= MHLEN);
6164 MH_ALIGN(m, len);
6165 } else {
6166 VERIFY(len <= MLEN);
6167 M_ALIGN(m, len);
6168 }
6169 m->m_len = len;
6170 return m;
6171 }
6172
6173 /*
6174 * Replacement for old M_PREPEND macro: allocate new mbuf to prepend to
6175 * chain, copy junk along, and adjust length.
6176 */
6177 struct mbuf *
6178 m_prepend_2(struct mbuf *m, int len, int how, int align)
6179 {
6180 if (M_LEADINGSPACE(m) >= len &&
6181 (!align || IS_P2ALIGNED((m->m_data - len), sizeof(u_int32_t)))) {
6182 m->m_data -= len;
6183 m->m_len += len;
6184 } else {
6185 m = m_prepend(m, len, how);
6186 }
6187 if ((m) && (m->m_flags & M_PKTHDR)) {
6188 m->m_pkthdr.len += len;
6189 }
6190 return m;
6191 }
6192
6193 /*
6194 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
6195 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf.
6196 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller.
6197 *
6198 * The last mbuf and offset accessed are passed in and adjusted on return to
6199 * avoid having to iterate over the entire mbuf chain each time.
6200 */
6201 struct mbuf *
6202 m_copym_mode(struct mbuf *m, int off0, int len0, int wait,
6203 struct mbuf **m_lastm, int *m_off, uint32_t mode)
6204 {
6205 struct mbuf *n, *mhdr = NULL, **np;
6206 int off = off0, len = len0;
6207 struct mbuf *top;
6208 int copyhdr = 0;
6209
6210 if (off < 0 || len < 0) {
6211 panic("m_copym: invalid offset %d or len %d", off, len);
6212 }
6213
6214 VERIFY((mode != M_COPYM_MUST_COPY_HDR &&
6215 mode != M_COPYM_MUST_MOVE_HDR) || (m->m_flags & M_PKTHDR));
6216
6217 if ((off == 0 && (m->m_flags & M_PKTHDR)) ||
6218 mode == M_COPYM_MUST_COPY_HDR || mode == M_COPYM_MUST_MOVE_HDR) {
6219 mhdr = m;
6220 copyhdr = 1;
6221 }
6222
6223 if (m_lastm != NULL && *m_lastm != NULL) {
6224 if (off0 >= *m_off) {
6225 m = *m_lastm;
6226 off = off0 - *m_off;
6227 }
6228 }
6229
6230 while (off >= m->m_len) {
6231 off -= m->m_len;
6232 m = m->m_next;
6233 }
6234 np = ⊤
6235 top = NULL;
6236
6237 while (len > 0) {
6238 if (m == NULL) {
6239 if (len != M_COPYALL) {
6240 panic("m_copym: len != M_COPYALL");
6241 }
6242 break;
6243 }
6244
6245 if (copyhdr) {
6246 n = _M_RETRYHDR(wait, m->m_type);
6247 } else {
6248 n = _M_RETRY(wait, m->m_type);
6249 }
6250 *np = n;
6251
6252 if (n == NULL) {
6253 goto nospace;
6254 }
6255
6256 if (copyhdr != 0) {
6257 if ((mode == M_COPYM_MOVE_HDR) ||
6258 (mode == M_COPYM_MUST_MOVE_HDR)) {
6259 M_COPY_PKTHDR(n, mhdr);
6260 } else if ((mode == M_COPYM_COPY_HDR) ||
6261 (mode == M_COPYM_MUST_COPY_HDR)) {
6262 if (m_dup_pkthdr(n, mhdr, wait) == 0) {
6263 goto nospace;
6264 }
6265 }
6266 if (len == M_COPYALL) {
6267 n->m_pkthdr.len -= off0;
6268 } else {
6269 n->m_pkthdr.len = len;
6270 }
6271 copyhdr = 0;
6272 /*
6273 * There is data to copy from the packet header mbuf
6274 * if it is empty or it is before the starting offset
6275 */
6276 if (mhdr != m) {
6277 np = &n->m_next;
6278 continue;
6279 }
6280 }
6281 n->m_len = MIN(len, (m->m_len - off));
6282 if (m->m_flags & M_EXT) {
6283 n->m_ext = m->m_ext;
6284 m_incref(m);
6285 n->m_data = m->m_data + off;
6286 n->m_flags |= M_EXT;
6287 } else {
6288 /*
6289 * Limit to the capacity of the destination
6290 */
6291 if (n->m_flags & M_PKTHDR) {
6292 n->m_len = MIN(n->m_len, MHLEN);
6293 } else {
6294 n->m_len = MIN(n->m_len, MLEN);
6295 }
6296
6297 if (MTOD(n, char *) + n->m_len > ((char *)n) + _MSIZE) {
6298 panic("%s n %p copy overflow",
6299 __func__, n);
6300 }
6301
6302 bcopy(MTOD(m, caddr_t) + off, MTOD(n, caddr_t),
6303 (unsigned)n->m_len);
6304 }
6305 if (len != M_COPYALL) {
6306 len -= n->m_len;
6307 }
6308
6309 if (len == 0) {
6310 if (m_lastm != NULL) {
6311 *m_lastm = m;
6312 *m_off = off0 + len0 - (off + n->m_len);
6313 }
6314 }
6315 off = 0;
6316 m = m->m_next;
6317 np = &n->m_next;
6318 }
6319
6320 return top;
6321 nospace:
6322 m_freem(top);
6323
6324 return NULL;
6325 }
6326
6327
6328 struct mbuf *
6329 m_copym(struct mbuf *m, int off0, int len, int wait)
6330 {
6331 return m_copym_mode(m, off0, len, wait, NULL, NULL, M_COPYM_MOVE_HDR);
6332 }
6333
6334 /*
6335 * Equivalent to m_copym except that all necessary mbuf hdrs are allocated
6336 * within this routine also.
6337 *
6338 * The last mbuf and offset accessed are passed in and adjusted on return to
6339 * avoid having to iterate over the entire mbuf chain each time.
6340 */
6341 struct mbuf *
6342 m_copym_with_hdrs(struct mbuf *m0, int off0, int len0, int wait,
6343 struct mbuf **m_lastm, int *m_off, uint32_t mode)
6344 {
6345 struct mbuf *m = m0, *n, **np = NULL;
6346 int off = off0, len = len0;
6347 struct mbuf *top = NULL;
6348 #if CONFIG_MBUF_MCACHE
6349 int mcflags = MSLEEPF(wait);
6350 mcache_obj_t *list = NULL;
6351 #else
6352 zstack_t list = {};
6353 #endif /* CONFIG_MBUF_MCACHE */
6354 int copyhdr = 0;
6355 int type = 0;
6356 int needed = 0;
6357
6358 if (off == 0 && (m->m_flags & M_PKTHDR)) {
6359 copyhdr = 1;
6360 }
6361
6362 if (m_lastm != NULL && *m_lastm != NULL) {
6363 if (off0 >= *m_off) {
6364 m = *m_lastm;
6365 off = off0 - *m_off;
6366 }
6367 }
6368
6369 while (off >= m->m_len) {
6370 off -= m->m_len;
6371 m = m->m_next;
6372 }
6373
6374 n = m;
6375 while (len > 0) {
6376 needed++;
6377 len -= MIN(len, (n->m_len - ((needed == 1) ? off : 0)));
6378 n = n->m_next;
6379 }
6380 needed++;
6381 len = len0;
6382
6383 #if CONFIG_MBUF_MCACHE
6384 /*
6385 * If the caller doesn't want to be put to sleep, mark it with
6386 * MCR_TRYHARD so that we may reclaim buffers from other places
6387 * before giving up.
6388 */
6389 if (mcflags & MCR_NOSLEEP) {
6390 mcflags |= MCR_TRYHARD;
6391 }
6392
6393 if (mcache_alloc_ext(m_cache(MC_MBUF), &list, needed,
6394 mcflags) != needed) {
6395 goto nospace;
6396 }
6397 #else
6398 list = mz_alloc_n(needed, wait);
6399 if (zstack_count(list) != needed) {
6400 goto nospace;
6401 }
6402 #endif /* CONFIG_MBUF_MCACHE */
6403
6404 needed = 0;
6405 while (len > 0) {
6406 #if CONFIG_MBUF_MCACHE
6407 n = (struct mbuf *)list;
6408 list = list->obj_next;
6409 #else
6410 n = zstack_pop(&list);
6411 #endif /* CONFIG_MBUF_MCACHE */
6412 ASSERT(n != NULL && m != NULL);
6413
6414 type = (top == NULL) ? MT_HEADER : m->m_type;
6415 MBUF_INIT(n, (top == NULL), type);
6416
6417 if (top == NULL) {
6418 top = n;
6419 np = &top->m_next;
6420 continue;
6421 } else {
6422 needed++;
6423 *np = n;
6424 }
6425
6426 if (copyhdr) {
6427 if ((mode == M_COPYM_MOVE_HDR) ||
6428 (mode == M_COPYM_MUST_MOVE_HDR)) {
6429 M_COPY_PKTHDR(n, m);
6430 } else if ((mode == M_COPYM_COPY_HDR) ||
6431 (mode == M_COPYM_MUST_COPY_HDR)) {
6432 if (m_dup_pkthdr(n, m, wait) == 0) {
6433 #if !CONFIG_MBUF_MCACHE
6434 m_elide(n);
6435 #endif
6436 goto nospace;
6437 }
6438 }
6439 n->m_pkthdr.len = len;
6440 copyhdr = 0;
6441 }
6442 n->m_len = MIN(len, (m->m_len - off));
6443
6444 if (m->m_flags & M_EXT) {
6445 n->m_ext = m->m_ext;
6446 m_incref(m);
6447 n->m_data = m->m_data + off;
6448 n->m_flags |= M_EXT;
6449 } else {
6450 if (m_mtod_end(n) > m_mtod_upper_bound(n)) {
6451 panic("%s n %p copy overflow",
6452 __func__, n);
6453 }
6454
6455 bcopy(MTOD(m, caddr_t) + off, MTOD(n, caddr_t),
6456 (unsigned)n->m_len);
6457 }
6458 len -= n->m_len;
6459
6460 if (len == 0) {
6461 if (m_lastm != NULL) {
6462 *m_lastm = m;
6463 *m_off = off0 + len0 - (off + n->m_len);
6464 }
6465 break;
6466 }
6467 off = 0;
6468 m = m->m_next;
6469 np = &n->m_next;
6470 }
6471
6472 mtype_stat_inc(MT_HEADER);
6473 mtype_stat_add(type, needed);
6474 mtype_stat_sub(MT_FREE, needed + 1);
6475
6476 #if CONFIG_MBUF_MCACHE
6477 ASSERT(list == NULL);
6478 #else
6479 ASSERT(zstack_empty(list));
6480 #endif /* CONFIG_MBUF_MCACHE */
6481
6482 return top;
6483
6484 nospace:
6485 #if CONFIG_MBUF_MCACHE
6486 if (list != NULL) {
6487 mcache_free_ext(m_cache(MC_MBUF), list);
6488 }
6489 #else
6490 if (!zstack_empty(list)) {
6491 /* No need to elide, these mbufs came from the cache. */
6492 mz_free_n(list);
6493 }
6494 #endif /* CONFIG_MBUF_MCACHE */
6495 if (top != NULL) {
6496 m_freem(top);
6497 }
6498 return NULL;
6499 }
6500
6501 /*
6502 * Copy data from an mbuf chain starting "off" bytes from the beginning,
6503 * continuing for "len" bytes, into the indicated buffer.
6504 */
6505 void
6506 m_copydata(struct mbuf *m, int off, int len, void *vp)
6507 {
6508 int off0 = off, len0 = len;
6509 struct mbuf *m0 = m;
6510 unsigned count;
6511 char *cp = vp;
6512
6513 if (__improbable(off < 0 || len < 0)) {
6514 panic("%s: invalid offset %d or len %d", __func__, off, len);
6515 /* NOTREACHED */
6516 }
6517
6518 while (off > 0) {
6519 if (__improbable(m == NULL)) {
6520 panic("%s: invalid mbuf chain %p [off %d, len %d]",
6521 __func__, m0, off0, len0);
6522 /* NOTREACHED */
6523 }
6524 if (off < m->m_len) {
6525 break;
6526 }
6527 off -= m->m_len;
6528 m = m->m_next;
6529 }
6530 while (len > 0) {
6531 if (__improbable(m == NULL)) {
6532 panic("%s: invalid mbuf chain %p [off %d, len %d]",
6533 __func__, m0, off0, len0);
6534 /* NOTREACHED */
6535 }
6536 count = MIN(m->m_len - off, len);
6537 bcopy(MTOD(m, caddr_t) + off, cp, count);
6538 len -= count;
6539 cp += count;
6540 off = 0;
6541 m = m->m_next;
6542 }
6543 }
6544
6545 /*
6546 * Concatenate mbuf chain n to m. Both chains must be of the same type
6547 * (e.g. MT_DATA). Any m_pkthdr is not updated.
6548 */
6549 void
6550 m_cat(struct mbuf *m, struct mbuf *n)
6551 {
6552 while (m->m_next) {
6553 m = m->m_next;
6554 }
6555 while (n) {
6556 if ((m->m_flags & M_EXT) ||
6557 m->m_data + m->m_len + n->m_len >= (uintptr_t)&m->m_dat[MLEN]) {
6558 /* just join the two chains */
6559 m->m_next = n;
6560 return;
6561 }
6562 /* splat the data from one into the other */
6563 bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
6564 (u_int)n->m_len);
6565 m->m_len += n->m_len;
6566 n = m_free(n);
6567 }
6568 }
6569
6570 void
6571 m_adj(struct mbuf *mp, int req_len)
6572 {
6573 int len = req_len;
6574 struct mbuf *m;
6575 int count;
6576
6577 if ((m = mp) == NULL) {
6578 return;
6579 }
6580 if (len >= 0) {
6581 /*
6582 * Trim from head.
6583 */
6584 while (m != NULL && len > 0) {
6585 if (m->m_len <= len) {
6586 len -= m->m_len;
6587 m->m_len = 0;
6588 m = m->m_next;
6589 } else {
6590 m->m_len -= len;
6591 m->m_data += len;
6592 len = 0;
6593 }
6594 }
6595 m = mp;
6596 if (m->m_flags & M_PKTHDR) {
6597 m->m_pkthdr.len -= (req_len - len);
6598 }
6599 } else {
6600 /*
6601 * Trim from tail. Scan the mbuf chain,
6602 * calculating its length and finding the last mbuf.
6603 * If the adjustment only affects this mbuf, then just
6604 * adjust and return. Otherwise, rescan and truncate
6605 * after the remaining size.
6606 */
6607 len = -len;
6608 count = 0;
6609 for (;;) {
6610 count += m->m_len;
6611 if (m->m_next == (struct mbuf *)0) {
6612 break;
6613 }
6614 m = m->m_next;
6615 }
6616 if (m->m_len >= len) {
6617 m->m_len -= len;
6618 m = mp;
6619 if (m->m_flags & M_PKTHDR) {
6620 m->m_pkthdr.len -= len;
6621 }
6622 return;
6623 }
6624 count -= len;
6625 if (count < 0) {
6626 count = 0;
6627 }
6628 /*
6629 * Correct length for chain is "count".
6630 * Find the mbuf with last data, adjust its length,
6631 * and toss data from remaining mbufs on chain.
6632 */
6633 m = mp;
6634 if (m->m_flags & M_PKTHDR) {
6635 m->m_pkthdr.len = count;
6636 }
6637 for (; m; m = m->m_next) {
6638 if (m->m_len >= count) {
6639 m->m_len = count;
6640 break;
6641 }
6642 count -= m->m_len;
6643 }
6644 while ((m = m->m_next)) {
6645 m->m_len = 0;
6646 }
6647 }
6648 }
6649
6650 /*
6651 * Rearange an mbuf chain so that len bytes are contiguous
6652 * and in the data area of an mbuf (so that mtod
6653 * will work for a structure of size len). Returns the resulting
6654 * mbuf chain on success, frees it and returns null on failure.
6655 * If there is room, it will add up to max_protohdr-len extra bytes to the
6656 * contiguous region in an attempt to avoid being called next time.
6657 */
6658 struct mbuf *
6659 m_pullup(struct mbuf *n, int len)
6660 {
6661 struct mbuf *m;
6662 int count;
6663 int space;
6664
6665 /* check invalid arguments */
6666 if (n == NULL) {
6667 panic("%s: n == NULL", __func__);
6668 }
6669 if (len < 0) {
6670 os_log_info(OS_LOG_DEFAULT, "%s: failed negative len %d",
6671 __func__, len);
6672 goto bad;
6673 }
6674 if (len > MLEN) {
6675 os_log_info(OS_LOG_DEFAULT, "%s: failed len %d too big",
6676 __func__, len);
6677 goto bad;
6678 }
6679 if ((n->m_flags & M_EXT) == 0 &&
6680 m_mtod_current(n) >= m_mtod_upper_bound(n)) {
6681 os_log_info(OS_LOG_DEFAULT, "%s: m_data out of bounds",
6682 __func__);
6683 goto bad;
6684 }
6685
6686 /*
6687 * If first mbuf has no cluster, and has room for len bytes
6688 * without shifting current data, pullup into it,
6689 * otherwise allocate a new mbuf to prepend to the chain.
6690 */
6691 if ((n->m_flags & M_EXT) == 0 &&
6692 len < m_mtod_upper_bound(n) - m_mtod_current(n) && n->m_next != NULL) {
6693 if (n->m_len >= len) {
6694 return n;
6695 }
6696 m = n;
6697 n = n->m_next;
6698 len -= m->m_len;
6699 } else {
6700 if (len > MHLEN) {
6701 goto bad;
6702 }
6703 _MGET(m, M_DONTWAIT, n->m_type);
6704 if (m == 0) {
6705 goto bad;
6706 }
6707 m->m_len = 0;
6708 if (n->m_flags & M_PKTHDR) {
6709 M_COPY_PKTHDR(m, n);
6710 n->m_flags &= ~M_PKTHDR;
6711 }
6712 }
6713 space = m_mtod_upper_bound(m) - m_mtod_end(m);
6714 do {
6715 count = MIN(MIN(MAX(len, max_protohdr), space), n->m_len);
6716 bcopy(MTOD(n, caddr_t), MTOD(m, caddr_t) + m->m_len,
6717 (unsigned)count);
6718 len -= count;
6719 m->m_len += count;
6720 n->m_len -= count;
6721 space -= count;
6722 if (n->m_len != 0) {
6723 n->m_data += count;
6724 } else {
6725 n = m_free(n);
6726 }
6727 } while (len > 0 && n != NULL);
6728 if (len > 0) {
6729 (void) m_free(m);
6730 goto bad;
6731 }
6732 m->m_next = n;
6733 return m;
6734 bad:
6735 m_freem(n);
6736 return 0;
6737 }
6738
6739 /*
6740 * Like m_pullup(), except a new mbuf is always allocated, and we allow
6741 * the amount of empty space before the data in the new mbuf to be specified
6742 * (in the event that the caller expects to prepend later).
6743 */
6744 __private_extern__ struct mbuf *
6745 m_copyup(struct mbuf *n, int len, int dstoff)
6746 {
6747 struct mbuf *m;
6748 int count, space;
6749
6750 VERIFY(len >= 0 && dstoff >= 0);
6751
6752 if (len > (MHLEN - dstoff)) {
6753 goto bad;
6754 }
6755 MGET(m, M_DONTWAIT, n->m_type);
6756 if (m == NULL) {
6757 goto bad;
6758 }
6759 m->m_len = 0;
6760 if (n->m_flags & M_PKTHDR) {
6761 m_copy_pkthdr(m, n);
6762 n->m_flags &= ~M_PKTHDR;
6763 }
6764 m->m_data += dstoff;
6765 space = m_mtod_upper_bound(m) - m_mtod_end(m);
6766 do {
6767 count = min(min(max(len, max_protohdr), space), n->m_len);
6768 memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
6769 (unsigned)count);
6770 len -= count;
6771 m->m_len += count;
6772 n->m_len -= count;
6773 space -= count;
6774 if (n->m_len) {
6775 n->m_data += count;
6776 } else {
6777 n = m_free(n);
6778 }
6779 } while (len > 0 && n);
6780 if (len > 0) {
6781 (void) m_free(m);
6782 goto bad;
6783 }
6784 m->m_next = n;
6785 return m;
6786 bad:
6787 m_freem(n);
6788
6789 return NULL;
6790 }
6791
6792 /*
6793 * Partition an mbuf chain in two pieces, returning the tail --
6794 * all but the first len0 bytes. In case of failure, it returns NULL and
6795 * attempts to restore the chain to its original state.
6796 */
6797 struct mbuf *
6798 m_split(struct mbuf *m0, int len0, int wait)
6799 {
6800 return m_split0(m0, len0, wait, 1);
6801 }
6802
6803 static struct mbuf *
6804 m_split0(struct mbuf *m0, int len0, int wait, int copyhdr)
6805 {
6806 struct mbuf *m, *n;
6807 unsigned len = len0, remain;
6808
6809 /*
6810 * First iterate to the mbuf which contains the first byte of
6811 * data at offset len0
6812 */
6813 for (m = m0; m && len > m->m_len; m = m->m_next) {
6814 len -= m->m_len;
6815 }
6816 if (m == NULL) {
6817 return NULL;
6818 }
6819 /*
6820 * len effectively is now the offset in the current
6821 * mbuf where we have to perform split.
6822 *
6823 * remain becomes the tail length.
6824 * Note that len can also be == m->m_len
6825 */
6826 remain = m->m_len - len;
6827
6828 /*
6829 * If current mbuf len contains the entire remaining offset len,
6830 * just make the second mbuf chain pointing to next mbuf onwards
6831 * and return after making necessary adjustments
6832 */
6833 if (copyhdr && (m0->m_flags & M_PKTHDR) && remain == 0) {
6834 _MGETHDR(n, wait, m0->m_type);
6835 if (n == NULL) {
6836 return NULL;
6837 }
6838 n->m_next = m->m_next;
6839 m->m_next = NULL;
6840 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
6841 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
6842 m0->m_pkthdr.len = len0;
6843 return n;
6844 }
6845 if (copyhdr && (m0->m_flags & M_PKTHDR)) {
6846 _MGETHDR(n, wait, m0->m_type);
6847 if (n == NULL) {
6848 return NULL;
6849 }
6850 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
6851 n->m_pkthdr.len = m0->m_pkthdr.len - len0;
6852 m0->m_pkthdr.len = len0;
6853
6854 /*
6855 * If current points to external storage
6856 * then it can be shared by making last mbuf
6857 * of head chain and first mbuf of current chain
6858 * pointing to different data offsets
6859 */
6860 if (m->m_flags & M_EXT) {
6861 goto extpacket;
6862 }
6863 if (remain > MHLEN) {
6864 /* m can't be the lead packet */
6865 MH_ALIGN(n, 0);
6866 n->m_next = m_split(m, len, wait);
6867 if (n->m_next == NULL) {
6868 (void) m_free(n);
6869 return NULL;
6870 } else {
6871 return n;
6872 }
6873 } else {
6874 MH_ALIGN(n, remain);
6875 }
6876 } else if (remain == 0) {
6877 n = m->m_next;
6878 m->m_next = NULL;
6879 return n;
6880 } else {
6881 _MGET(n, wait, m->m_type);
6882 if (n == NULL) {
6883 return NULL;
6884 }
6885
6886 if ((m->m_flags & M_EXT) == 0) {
6887 VERIFY(remain <= MLEN);
6888 M_ALIGN(n, remain);
6889 }
6890 }
6891 extpacket:
6892 if (m->m_flags & M_EXT) {
6893 n->m_flags |= M_EXT;
6894 n->m_ext = m->m_ext;
6895 m_incref(m);
6896 n->m_data = m->m_data + len;
6897 } else {
6898 bcopy(MTOD(m, caddr_t) + len, MTOD(n, caddr_t), remain);
6899 }
6900 n->m_len = remain;
6901 m->m_len = len;
6902 n->m_next = m->m_next;
6903 m->m_next = NULL;
6904 return n;
6905 }
6906
6907 /*
6908 * Routine to copy from device local memory into mbufs.
6909 */
6910 struct mbuf *
6911 m_devget(char *buf, int totlen, int off0, struct ifnet *ifp,
6912 void (*copy)(const void *, void *, size_t))
6913 {
6914 struct mbuf *m;
6915 struct mbuf *top = NULL, **mp = ⊤
6916 int off = off0, len;
6917 char *cp;
6918 char *epkt;
6919
6920 cp = buf;
6921 epkt = cp + totlen;
6922 if (off) {
6923 /*
6924 * If 'off' is non-zero, packet is trailer-encapsulated,
6925 * so we have to skip the type and length fields.
6926 */
6927 cp += off + 2 * sizeof(u_int16_t);
6928 totlen -= 2 * sizeof(u_int16_t);
6929 }
6930 _MGETHDR(m, M_DONTWAIT, MT_DATA);
6931 if (m == NULL) {
6932 return NULL;
6933 }
6934 m->m_pkthdr.rcvif = ifp;
6935 m->m_pkthdr.len = totlen;
6936 m->m_len = MHLEN;
6937
6938 while (totlen > 0) {
6939 if (top != NULL) {
6940 _MGET(m, M_DONTWAIT, MT_DATA);
6941 if (m == NULL) {
6942 m_freem(top);
6943 return NULL;
6944 }
6945 m->m_len = MLEN;
6946 }
6947 len = MIN(totlen, epkt - cp);
6948 if (len >= MINCLSIZE) {
6949 MCLGET(m, M_DONTWAIT);
6950 if (m->m_flags & M_EXT) {
6951 m->m_len = len = MIN(len, m_maxsize(MC_CL));
6952 } else {
6953 /* give up when it's out of cluster mbufs */
6954 if (top != NULL) {
6955 m_freem(top);
6956 }
6957 m_freem(m);
6958 return NULL;
6959 }
6960 } else {
6961 /*
6962 * Place initial small packet/header at end of mbuf.
6963 */
6964 if (len < m->m_len) {
6965 if (top == NULL &&
6966 len + max_linkhdr <= m->m_len) {
6967 m->m_data += max_linkhdr;
6968 }
6969 m->m_len = len;
6970 } else {
6971 len = m->m_len;
6972 }
6973 }
6974 if (copy) {
6975 copy(cp, MTOD(m, caddr_t), (unsigned)len);
6976 } else {
6977 bcopy(cp, MTOD(m, caddr_t), (unsigned)len);
6978 }
6979 cp += len;
6980 *mp = m;
6981 mp = &m->m_next;
6982 totlen -= len;
6983 if (cp == epkt) {
6984 cp = buf;
6985 }
6986 }
6987 return top;
6988 }
6989
6990 #if CONFIG_MBUF_MCACHE
6991 #ifndef MBUF_GROWTH_NORMAL_THRESH
6992 #define MBUF_GROWTH_NORMAL_THRESH 25
6993 #endif
6994
6995 /*
6996 * Cluster freelist allocation check.
6997 */
6998 static int
6999 m_howmany(int num, size_t bufsize)
7000 {
7001 int i = 0, j = 0;
7002 u_int32_t m_mbclusters, m_clusters, m_bigclusters, m_16kclusters;
7003 u_int32_t m_mbfree, m_clfree, m_bigclfree, m_16kclfree;
7004 u_int32_t sumclusters, freeclusters;
7005 u_int32_t percent_pool, percent_kmem;
7006 u_int32_t mb_growth, mb_growth_thresh;
7007
7008 VERIFY(bufsize == m_maxsize(MC_BIGCL) ||
7009 bufsize == m_maxsize(MC_16KCL));
7010
7011 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
7012
7013 /* Numbers in 2K cluster units */
7014 m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT;
7015 m_clusters = m_total(MC_CL);
7016 m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT;
7017 m_16kclusters = m_total(MC_16KCL);
7018 sumclusters = m_mbclusters + m_clusters + m_bigclusters;
7019
7020 m_mbfree = m_infree(MC_MBUF) >> NMBPCLSHIFT;
7021 m_clfree = m_infree(MC_CL);
7022 m_bigclfree = m_infree(MC_BIGCL) << NCLPBGSHIFT;
7023 m_16kclfree = m_infree(MC_16KCL);
7024 freeclusters = m_mbfree + m_clfree + m_bigclfree;
7025
7026 /* Bail if we've maxed out the mbuf memory map */
7027 if ((bufsize == m_maxsize(MC_BIGCL) && sumclusters >= nclusters) ||
7028 (njcl > 0 && bufsize == m_maxsize(MC_16KCL) &&
7029 (m_16kclusters << NCLPJCLSHIFT) >= njcl)) {
7030 mbwdog_logger("maxed out nclusters (%u >= %u) or njcl (%u >= %u)",
7031 sumclusters, nclusters,
7032 (m_16kclusters << NCLPJCLSHIFT), njcl);
7033 return 0;
7034 }
7035
7036 if (bufsize == m_maxsize(MC_BIGCL)) {
7037 /* Under minimum */
7038 if (m_bigclusters < m_minlimit(MC_BIGCL)) {
7039 return m_minlimit(MC_BIGCL) - m_bigclusters;
7040 }
7041
7042 percent_pool =
7043 ((sumclusters - freeclusters) * 100) / sumclusters;
7044 percent_kmem = (sumclusters * 100) / nclusters;
7045
7046 /*
7047 * If a light/normal user, grow conservatively (75%)
7048 * If a heavy user, grow aggressively (50%)
7049 */
7050 if (percent_kmem < MBUF_GROWTH_NORMAL_THRESH) {
7051 mb_growth = MB_GROWTH_NORMAL;
7052 } else {
7053 mb_growth = MB_GROWTH_AGGRESSIVE;
7054 }
7055
7056 if (percent_kmem < 5) {
7057 /* For initial allocations */
7058 i = num;
7059 } else {
7060 /* Return if >= MBIGCL_LOWAT clusters available */
7061 if (m_infree(MC_BIGCL) >= MBIGCL_LOWAT &&
7062 m_total(MC_BIGCL) >=
7063 MBIGCL_LOWAT + m_minlimit(MC_BIGCL)) {
7064 return 0;
7065 }
7066
7067 /* Ensure at least num clusters are accessible */
7068 if (num >= m_infree(MC_BIGCL)) {
7069 i = num - m_infree(MC_BIGCL);
7070 }
7071 if (num > m_total(MC_BIGCL) - m_minlimit(MC_BIGCL)) {
7072 j = num - (m_total(MC_BIGCL) -
7073 m_minlimit(MC_BIGCL));
7074 }
7075
7076 i = MAX(i, j);
7077
7078 /*
7079 * Grow pool if percent_pool > 75 (normal growth)
7080 * or percent_pool > 50 (aggressive growth).
7081 */
7082 mb_growth_thresh = 100 - (100 / (1 << mb_growth));
7083 if (percent_pool > mb_growth_thresh) {
7084 j = ((sumclusters + num) >> mb_growth) -
7085 freeclusters;
7086 }
7087 i = MAX(i, j);
7088 }
7089
7090 /* Check to ensure we didn't go over limits */
7091 if (i + m_bigclusters >= m_maxlimit(MC_BIGCL)) {
7092 i = m_maxlimit(MC_BIGCL) - m_bigclusters;
7093 }
7094 if ((i << 1) + sumclusters >= nclusters) {
7095 i = (nclusters - sumclusters) >> 1;
7096 }
7097 VERIFY((m_total(MC_BIGCL) + i) <= m_maxlimit(MC_BIGCL));
7098 VERIFY(sumclusters + (i << 1) <= nclusters);
7099 } else { /* 16K CL */
7100 VERIFY(njcl > 0);
7101 /* Ensure at least num clusters are available */
7102 if (num >= m_16kclfree) {
7103 i = num - m_16kclfree;
7104 }
7105
7106 /* Always grow 16KCL pool aggressively */
7107 if (((m_16kclusters + num) >> 1) > m_16kclfree) {
7108 j = ((m_16kclusters + num) >> 1) - m_16kclfree;
7109 }
7110 i = MAX(i, j);
7111
7112 /* Check to ensure we don't go over limit */
7113 if ((i + m_total(MC_16KCL)) >= m_maxlimit(MC_16KCL)) {
7114 i = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
7115 }
7116 }
7117 return i;
7118 }
7119 #endif /* CONFIG_MBUF_MCACHE */
7120 /*
7121 * Return the number of bytes in the mbuf chain, m.
7122 */
7123 unsigned int
7124 m_length(struct mbuf *m)
7125 {
7126 struct mbuf *m0;
7127 unsigned int pktlen;
7128
7129 if (m->m_flags & M_PKTHDR) {
7130 return m->m_pkthdr.len;
7131 }
7132
7133 pktlen = 0;
7134 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
7135 pktlen += m0->m_len;
7136 }
7137 return pktlen;
7138 }
7139
7140 /*
7141 * Copy data from a buffer back into the indicated mbuf chain,
7142 * starting "off" bytes from the beginning, extending the mbuf
7143 * chain if necessary.
7144 */
7145 void
7146 m_copyback(struct mbuf *m0, int off, int len, const void *cp)
7147 {
7148 #if DEBUG
7149 struct mbuf *origm = m0;
7150 int error;
7151 #endif /* DEBUG */
7152
7153 if (m0 == NULL) {
7154 return;
7155 }
7156
7157 #if DEBUG
7158 error =
7159 #endif /* DEBUG */
7160 m_copyback0(&m0, off, len, cp,
7161 M_COPYBACK0_COPYBACK | M_COPYBACK0_EXTEND, M_DONTWAIT);
7162
7163 #if DEBUG
7164 if (error != 0 || (m0 != NULL && origm != m0)) {
7165 panic("m_copyback");
7166 }
7167 #endif /* DEBUG */
7168 }
7169
7170 struct mbuf *
7171 m_copyback_cow(struct mbuf *m0, int off, int len, const void *cp, int how)
7172 {
7173 int error;
7174
7175 /* don't support chain expansion */
7176 VERIFY(off + len <= m_length(m0));
7177
7178 error = m_copyback0(&m0, off, len, cp,
7179 M_COPYBACK0_COPYBACK | M_COPYBACK0_COW, how);
7180 if (error) {
7181 /*
7182 * no way to recover from partial success.
7183 * just free the chain.
7184 */
7185 m_freem(m0);
7186 return NULL;
7187 }
7188 return m0;
7189 }
7190
7191 /*
7192 * m_makewritable: ensure the specified range writable.
7193 */
7194 int
7195 m_makewritable(struct mbuf **mp, int off, int len, int how)
7196 {
7197 int error;
7198 #if DEBUG
7199 struct mbuf *n;
7200 int origlen, reslen;
7201
7202 origlen = m_length(*mp);
7203 #endif /* DEBUG */
7204
7205 #if 0 /* M_COPYALL is large enough */
7206 if (len == M_COPYALL) {
7207 len = m_length(*mp) - off; /* XXX */
7208 }
7209 #endif
7210
7211 error = m_copyback0(mp, off, len, NULL,
7212 M_COPYBACK0_PRESERVE | M_COPYBACK0_COW, how);
7213
7214 #if DEBUG
7215 reslen = 0;
7216 for (n = *mp; n; n = n->m_next) {
7217 reslen += n->m_len;
7218 }
7219 if (origlen != reslen) {
7220 panic("m_makewritable: length changed");
7221 }
7222 if (((*mp)->m_flags & M_PKTHDR) && reslen != (*mp)->m_pkthdr.len) {
7223 panic("m_makewritable: inconsist");
7224 }
7225 #endif /* DEBUG */
7226
7227 return error;
7228 }
7229
7230 static int
7231 m_copyback0(struct mbuf **mp0, int off, int len, const void *vp, int flags,
7232 int how)
7233 {
7234 int mlen;
7235 struct mbuf *m, *n;
7236 struct mbuf **mp;
7237 int totlen = 0;
7238 const char *cp = vp;
7239
7240 VERIFY(mp0 != NULL);
7241 VERIFY(*mp0 != NULL);
7242 VERIFY((flags & M_COPYBACK0_PRESERVE) == 0 || cp == NULL);
7243 VERIFY((flags & M_COPYBACK0_COPYBACK) == 0 || cp != NULL);
7244
7245 /*
7246 * we don't bother to update "totlen" in the case of M_COPYBACK0_COW,
7247 * assuming that M_COPYBACK0_EXTEND and M_COPYBACK0_COW are exclusive.
7248 */
7249
7250 VERIFY((~flags & (M_COPYBACK0_EXTEND | M_COPYBACK0_COW)) != 0);
7251
7252 mp = mp0;
7253 m = *mp;
7254 while (off > (mlen = m->m_len)) {
7255 off -= mlen;
7256 totlen += mlen;
7257 if (m->m_next == NULL) {
7258 int tspace;
7259 extend:
7260 if (!(flags & M_COPYBACK0_EXTEND)) {
7261 goto out;
7262 }
7263
7264 /*
7265 * try to make some space at the end of "m".
7266 */
7267
7268 mlen = m->m_len;
7269 if (off + len >= MINCLSIZE &&
7270 !(m->m_flags & M_EXT) && m->m_len == 0) {
7271 MCLGET(m, how);
7272 }
7273 tspace = M_TRAILINGSPACE(m);
7274 if (tspace > 0) {
7275 tspace = MIN(tspace, off + len);
7276 VERIFY(tspace > 0);
7277 bzero(mtod(m, char *) + m->m_len,
7278 MIN(off, tspace));
7279 m->m_len += tspace;
7280 off += mlen;
7281 totlen -= mlen;
7282 continue;
7283 }
7284
7285 /*
7286 * need to allocate an mbuf.
7287 */
7288
7289 if (off + len >= MINCLSIZE) {
7290 n = m_getcl(how, m->m_type, 0);
7291 } else {
7292 n = _M_GET(how, m->m_type);
7293 }
7294 if (n == NULL) {
7295 goto out;
7296 }
7297 n->m_len = 0;
7298 n->m_len = MIN(M_TRAILINGSPACE(n), off + len);
7299 bzero(mtod(n, char *), MIN(n->m_len, off));
7300 m->m_next = n;
7301 }
7302 mp = &m->m_next;
7303 m = m->m_next;
7304 }
7305 while (len > 0) {
7306 mlen = m->m_len - off;
7307 if (mlen != 0 && m_mclhasreference(m)) {
7308 char *datap;
7309 int eatlen;
7310
7311 /*
7312 * this mbuf is read-only.
7313 * allocate a new writable mbuf and try again.
7314 */
7315
7316 #if DIAGNOSTIC
7317 if (!(flags & M_COPYBACK0_COW)) {
7318 panic("m_copyback0: read-only");
7319 }
7320 #endif /* DIAGNOSTIC */
7321
7322 /*
7323 * if we're going to write into the middle of
7324 * a mbuf, split it first.
7325 */
7326 if (off > 0 && len < mlen) {
7327 n = m_split0(m, off, how, 0);
7328 if (n == NULL) {
7329 goto enobufs;
7330 }
7331 m->m_next = n;
7332 mp = &m->m_next;
7333 m = n;
7334 off = 0;
7335 continue;
7336 }
7337
7338 /*
7339 * XXX TODO coalesce into the trailingspace of
7340 * the previous mbuf when possible.
7341 */
7342
7343 /*
7344 * allocate a new mbuf. copy packet header if needed.
7345 */
7346 n = _M_GET(how, m->m_type);
7347 if (n == NULL) {
7348 goto enobufs;
7349 }
7350 if (off == 0 && (m->m_flags & M_PKTHDR)) {
7351 M_COPY_PKTHDR(n, m);
7352 n->m_len = MHLEN;
7353 } else {
7354 if (len >= MINCLSIZE) {
7355 MCLGET(n, M_DONTWAIT);
7356 }
7357 n->m_len =
7358 (n->m_flags & M_EXT) ? MCLBYTES : MLEN;
7359 }
7360 if (n->m_len > len) {
7361 n->m_len = len;
7362 }
7363
7364 /*
7365 * free the region which has been overwritten.
7366 * copying data from old mbufs if requested.
7367 */
7368 if (flags & M_COPYBACK0_PRESERVE) {
7369 datap = mtod(n, char *);
7370 } else {
7371 datap = NULL;
7372 }
7373 eatlen = n->m_len;
7374 VERIFY(off == 0 || eatlen >= mlen);
7375 if (off > 0) {
7376 VERIFY(len >= mlen);
7377 m->m_len = off;
7378 m->m_next = n;
7379 if (datap) {
7380 m_copydata(m, off, mlen, datap);
7381 datap += mlen;
7382 }
7383 eatlen -= mlen;
7384 mp = &m->m_next;
7385 m = m->m_next;
7386 }
7387 while (m != NULL && m_mclhasreference(m) &&
7388 n->m_type == m->m_type && eatlen > 0) {
7389 mlen = MIN(eatlen, m->m_len);
7390 if (datap) {
7391 m_copydata(m, 0, mlen, datap);
7392 datap += mlen;
7393 }
7394 m->m_data += mlen;
7395 m->m_len -= mlen;
7396 eatlen -= mlen;
7397 if (m->m_len == 0) {
7398 *mp = m = m_free(m);
7399 }
7400 }
7401 if (eatlen > 0) {
7402 n->m_len -= eatlen;
7403 }
7404 n->m_next = m;
7405 *mp = m = n;
7406 continue;
7407 }
7408 mlen = MIN(mlen, len);
7409 if (flags & M_COPYBACK0_COPYBACK) {
7410 bcopy(cp, mtod(m, caddr_t) + off, (unsigned)mlen);
7411 cp += mlen;
7412 }
7413 len -= mlen;
7414 mlen += off;
7415 off = 0;
7416 totlen += mlen;
7417 if (len == 0) {
7418 break;
7419 }
7420 if (m->m_next == NULL) {
7421 goto extend;
7422 }
7423 mp = &m->m_next;
7424 m = m->m_next;
7425 }
7426 out:
7427 if (((m = *mp0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) {
7428 VERIFY(flags & M_COPYBACK0_EXTEND);
7429 m->m_pkthdr.len = totlen;
7430 }
7431
7432 return 0;
7433
7434 enobufs:
7435 return ENOBUFS;
7436 }
7437
7438 uint64_t
7439 mcl_to_paddr(char *addr)
7440 {
7441 #if CONFIG_MBUF_MCACHE
7442 vm_offset_t base_phys;
7443
7444 if (!MBUF_IN_MAP(addr)) {
7445 return 0;
7446 }
7447 base_phys = mcl_paddr[atop_64(addr - (char *)mbutl)];
7448
7449 if (base_phys == 0) {
7450 return 0;
7451 }
7452 return (uint64_t)(ptoa_64(base_phys) | ((uint64_t)addr & PAGE_MASK));
7453 #else
7454 extern addr64_t kvtophys(vm_offset_t va);
7455
7456 return kvtophys((vm_offset_t)addr);
7457 #endif /* CONFIG_MBUF_MCACHE */
7458 }
7459
7460 /*
7461 * Dup the mbuf chain passed in. The whole thing. No cute additional cruft.
7462 * And really copy the thing. That way, we don't "precompute" checksums
7463 * for unsuspecting consumers. Assumption: m->m_nextpkt == 0. Trick: for
7464 * small packets, don't dup into a cluster. That way received packets
7465 * don't take up too much room in the sockbuf (cf. sbspace()).
7466 */
7467 struct mbuf *
7468 m_dup(struct mbuf *m, int how)
7469 {
7470 struct mbuf *n, **np;
7471 struct mbuf *top;
7472 int copyhdr = 0;
7473
7474 np = ⊤
7475 top = NULL;
7476 if (m->m_flags & M_PKTHDR) {
7477 copyhdr = 1;
7478 }
7479
7480 /*
7481 * Quick check: if we have one mbuf and its data fits in an
7482 * mbuf with packet header, just copy and go.
7483 */
7484 if (m->m_next == NULL) {
7485 /* Then just move the data into an mbuf and be done... */
7486 if (copyhdr) {
7487 if (m->m_pkthdr.len <= MHLEN && m->m_len <= MHLEN) {
7488 if ((n = _M_GETHDR(how, m->m_type)) == NULL) {
7489 return NULL;
7490 }
7491 n->m_len = m->m_len;
7492 m_dup_pkthdr(n, m, how);
7493 bcopy(MTOD(m, caddr_t), MTOD(n, caddr_t), m->m_len);
7494 return n;
7495 }
7496 } else if (m->m_len <= MLEN) {
7497 if ((n = _M_GET(how, m->m_type)) == NULL) {
7498 return NULL;
7499 }
7500 bcopy(MTOD(m, caddr_t), MTOD(n, caddr_t), m->m_len);
7501 n->m_len = m->m_len;
7502 return n;
7503 }
7504 }
7505 while (m != NULL) {
7506 #if BLUE_DEBUG
7507 printf("<%x: %x, %x, %x\n", m, m->m_flags, m->m_len,
7508 m->m_data);
7509 #endif
7510 if (copyhdr) {
7511 n = _M_GETHDR(how, m->m_type);
7512 } else {
7513 n = _M_GET(how, m->m_type);
7514 }
7515 if (n == NULL) {
7516 goto nospace;
7517 }
7518 if (m->m_flags & M_EXT) {
7519 if (m->m_len <= m_maxsize(MC_CL)) {
7520 MCLGET(n, how);
7521 } else if (m->m_len <= m_maxsize(MC_BIGCL)) {
7522 n = m_mbigget(n, how);
7523 } else if (m->m_len <= m_maxsize(MC_16KCL) && njcl > 0) {
7524 n = m_m16kget(n, how);
7525 }
7526 if (!(n->m_flags & M_EXT)) {
7527 (void) m_free(n);
7528 goto nospace;
7529 }
7530 } else {
7531 VERIFY((copyhdr == 1 && m->m_len <= MHLEN) ||
7532 (copyhdr == 0 && m->m_len <= MLEN));
7533 }
7534 *np = n;
7535 if (copyhdr) {
7536 /* Don't use M_COPY_PKTHDR: preserve m_data */
7537 m_dup_pkthdr(n, m, how);
7538 copyhdr = 0;
7539 if (!(n->m_flags & M_EXT)) {
7540 n->m_data = (uintptr_t)n->m_pktdat;
7541 }
7542 }
7543 n->m_len = m->m_len;
7544 /*
7545 * Get the dup on the same bdry as the original
7546 * Assume that the two mbufs have the same offset to data area
7547 * (up to word boundaries)
7548 */
7549 bcopy(MTOD(m, caddr_t), MTOD(n, caddr_t), (unsigned)n->m_len);
7550 m = m->m_next;
7551 np = &n->m_next;
7552 #if BLUE_DEBUG
7553 printf(">%x: %x, %x, %x\n", n, n->m_flags, n->m_len,
7554 n->m_data);
7555 #endif
7556 }
7557
7558 return top;
7559
7560 nospace:
7561 m_freem(top);
7562 return NULL;
7563 }
7564
7565 #define MBUF_MULTIPAGES(m) \
7566 (((m)->m_flags & M_EXT) && \
7567 ((IS_P2ALIGNED((m)->m_data, PAGE_SIZE) \
7568 && (m)->m_len > PAGE_SIZE) || \
7569 (!IS_P2ALIGNED((m)->m_data, PAGE_SIZE) && \
7570 P2ROUNDUP((m)->m_data, PAGE_SIZE) < ((uintptr_t)(m)->m_data + (m)->m_len))))
7571
7572 static struct mbuf *
7573 m_expand(struct mbuf *m, struct mbuf **last)
7574 {
7575 struct mbuf *top = NULL;
7576 struct mbuf **nm = ⊤
7577 uintptr_t data0, data;
7578 unsigned int len0, len;
7579
7580 VERIFY(MBUF_MULTIPAGES(m));
7581 VERIFY(m->m_next == NULL);
7582 data0 = (uintptr_t)m->m_data;
7583 len0 = m->m_len;
7584 *last = top;
7585
7586 for (;;) {
7587 struct mbuf *n;
7588
7589 data = data0;
7590 if (IS_P2ALIGNED(data, PAGE_SIZE) && len0 > PAGE_SIZE) {
7591 len = PAGE_SIZE;
7592 } else if (!IS_P2ALIGNED(data, PAGE_SIZE) &&
7593 P2ROUNDUP(data, PAGE_SIZE) < (data + len0)) {
7594 len = P2ROUNDUP(data, PAGE_SIZE) - data;
7595 } else {
7596 len = len0;
7597 }
7598
7599 VERIFY(len > 0);
7600 VERIFY(m->m_flags & M_EXT);
7601 m->m_data = data;
7602 m->m_len = len;
7603
7604 *nm = *last = m;
7605 nm = &m->m_next;
7606 m->m_next = NULL;
7607
7608 data0 += len;
7609 len0 -= len;
7610 if (len0 == 0) {
7611 break;
7612 }
7613
7614 n = _M_RETRY(M_DONTWAIT, MT_DATA);
7615 if (n == NULL) {
7616 m_freem(top);
7617 top = *last = NULL;
7618 break;
7619 }
7620
7621 n->m_ext = m->m_ext;
7622 m_incref(m);
7623 n->m_flags |= M_EXT;
7624 m = n;
7625 }
7626 return top;
7627 }
7628
7629 struct mbuf *
7630 m_normalize(struct mbuf *m)
7631 {
7632 struct mbuf *top = NULL;
7633 struct mbuf **nm = ⊤
7634 boolean_t expanded = FALSE;
7635
7636 while (m != NULL) {
7637 struct mbuf *n;
7638
7639 n = m->m_next;
7640 m->m_next = NULL;
7641
7642 /* Does the data cross one or more page boundaries? */
7643 if (MBUF_MULTIPAGES(m)) {
7644 struct mbuf *last;
7645 if ((m = m_expand(m, &last)) == NULL) {
7646 m_freem(n);
7647 m_freem(top);
7648 top = NULL;
7649 break;
7650 }
7651 *nm = m;
7652 nm = &last->m_next;
7653 expanded = TRUE;
7654 } else {
7655 *nm = m;
7656 nm = &m->m_next;
7657 }
7658 m = n;
7659 }
7660 if (expanded) {
7661 os_atomic_inc(&mb_normalized, relaxed);
7662 }
7663 return top;
7664 }
7665
7666 /*
7667 * Append the specified data to the indicated mbuf chain,
7668 * Extend the mbuf chain if the new data does not fit in
7669 * existing space.
7670 *
7671 * Return 1 if able to complete the job; otherwise 0.
7672 */
7673 int
7674 m_append(struct mbuf *m0, int len, caddr_t cp)
7675 {
7676 struct mbuf *m, *n;
7677 int remainder, space;
7678
7679 for (m = m0; m->m_next != NULL; m = m->m_next) {
7680 ;
7681 }
7682 remainder = len;
7683 space = M_TRAILINGSPACE(m);
7684 if (space > 0) {
7685 /*
7686 * Copy into available space.
7687 */
7688 if (space > remainder) {
7689 space = remainder;
7690 }
7691 bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
7692 m->m_len += space;
7693 cp += space;
7694 remainder -= space;
7695 }
7696 while (remainder > 0) {
7697 /*
7698 * Allocate a new mbuf; could check space
7699 * and allocate a cluster instead.
7700 */
7701 n = m_get(M_WAITOK, m->m_type);
7702 if (n == NULL) {
7703 break;
7704 }
7705 n->m_len = min(MLEN, remainder);
7706 bcopy(cp, mtod(n, caddr_t), n->m_len);
7707 cp += n->m_len;
7708 remainder -= n->m_len;
7709 m->m_next = n;
7710 m = n;
7711 }
7712 if (m0->m_flags & M_PKTHDR) {
7713 m0->m_pkthdr.len += len - remainder;
7714 }
7715 return remainder == 0;
7716 }
7717
7718 struct mbuf *
7719 m_last(struct mbuf *m)
7720 {
7721 while (m->m_next != NULL) {
7722 m = m->m_next;
7723 }
7724 return m;
7725 }
7726
7727 unsigned int
7728 m_fixhdr(struct mbuf *m0)
7729 {
7730 u_int len;
7731
7732 VERIFY(m0->m_flags & M_PKTHDR);
7733
7734 len = m_length2(m0, NULL);
7735 m0->m_pkthdr.len = len;
7736 return len;
7737 }
7738
7739 unsigned int
7740 m_length2(struct mbuf *m0, struct mbuf **last)
7741 {
7742 struct mbuf *m;
7743 u_int len;
7744
7745 len = 0;
7746 for (m = m0; m != NULL; m = m->m_next) {
7747 len += m->m_len;
7748 if (m->m_next == NULL) {
7749 break;
7750 }
7751 }
7752 if (last != NULL) {
7753 *last = m;
7754 }
7755 return len;
7756 }
7757
7758 /*
7759 * Defragment a mbuf chain, returning the shortest possible chain of mbufs
7760 * and clusters. If allocation fails and this cannot be completed, NULL will
7761 * be returned, but the passed in chain will be unchanged. Upon success,
7762 * the original chain will be freed, and the new chain will be returned.
7763 *
7764 * If a non-packet header is passed in, the original mbuf (chain?) will
7765 * be returned unharmed.
7766 *
7767 * If offset is specfied, the first mbuf in the chain will have a leading
7768 * space of the amount stated by the "off" parameter.
7769 *
7770 * This routine requires that the m_pkthdr.header field of the original
7771 * mbuf chain is cleared by the caller.
7772 */
7773 struct mbuf *
7774 m_defrag_offset(struct mbuf *m0, u_int32_t off, int how)
7775 {
7776 struct mbuf *m_new = NULL, *m_final = NULL;
7777 int progress = 0, length, pktlen;
7778
7779 if (!(m0->m_flags & M_PKTHDR)) {
7780 return m0;
7781 }
7782
7783 VERIFY(off < MHLEN);
7784 m_fixhdr(m0); /* Needed sanity check */
7785
7786 pktlen = m0->m_pkthdr.len + off;
7787 if (pktlen > MHLEN) {
7788 m_final = m_getcl(how, MT_DATA, M_PKTHDR);
7789 } else {
7790 m_final = m_gethdr(how, MT_DATA);
7791 }
7792
7793 if (m_final == NULL) {
7794 goto nospace;
7795 }
7796
7797 if (off > 0) {
7798 pktlen -= off;
7799 m_final->m_data += off;
7800 }
7801
7802 /*
7803 * Caller must have handled the contents pointed to by this
7804 * pointer before coming here, as otherwise it will point to
7805 * the original mbuf which will get freed upon success.
7806 */
7807 VERIFY(m0->m_pkthdr.pkt_hdr == NULL);
7808
7809 if (m_dup_pkthdr(m_final, m0, how) == 0) {
7810 goto nospace;
7811 }
7812
7813 m_new = m_final;
7814
7815 while (progress < pktlen) {
7816 length = pktlen - progress;
7817 if (length > MCLBYTES) {
7818 length = MCLBYTES;
7819 }
7820 length -= ((m_new == m_final) ? off : 0);
7821 if (length < 0) {
7822 goto nospace;
7823 }
7824
7825 if (m_new == NULL) {
7826 if (length > MLEN) {
7827 m_new = m_getcl(how, MT_DATA, 0);
7828 } else {
7829 m_new = m_get(how, MT_DATA);
7830 }
7831 if (m_new == NULL) {
7832 goto nospace;
7833 }
7834 }
7835
7836 m_copydata(m0, progress, length, mtod(m_new, caddr_t));
7837 progress += length;
7838 m_new->m_len = length;
7839 if (m_new != m_final) {
7840 m_cat(m_final, m_new);
7841 }
7842 m_new = NULL;
7843 }
7844 m_freem(m0);
7845 m0 = m_final;
7846 return m0;
7847 nospace:
7848 if (m_final) {
7849 m_freem(m_final);
7850 }
7851 return NULL;
7852 }
7853
7854 struct mbuf *
7855 m_defrag(struct mbuf *m0, int how)
7856 {
7857 return m_defrag_offset(m0, 0, how);
7858 }
7859
7860 void
7861 m_mchtype(struct mbuf *m, int t)
7862 {
7863 mtype_stat_inc(t);
7864 mtype_stat_dec(m->m_type);
7865 (m)->m_type = t;
7866 }
7867
7868 void *__unsafe_indexable
7869 m_mtod(struct mbuf *m)
7870 {
7871 return m_mtod_current(m);
7872 }
7873
7874 void
7875 m_mcheck(struct mbuf *m)
7876 {
7877 _MCHECK(m);
7878 }
7879
7880 /*
7881 * Return a pointer to mbuf/offset of location in mbuf chain.
7882 */
7883 struct mbuf *
7884 m_getptr(struct mbuf *m, int loc, int *off)
7885 {
7886 while (loc >= 0) {
7887 /* Normal end of search. */
7888 if (m->m_len > loc) {
7889 *off = loc;
7890 return m;
7891 } else {
7892 loc -= m->m_len;
7893 if (m->m_next == NULL) {
7894 if (loc == 0) {
7895 /* Point at the end of valid data. */
7896 *off = m->m_len;
7897 return m;
7898 }
7899 return NULL;
7900 }
7901 m = m->m_next;
7902 }
7903 }
7904 return NULL;
7905 }
7906
7907 #if CONFIG_MBUF_MCACHE
7908 /*
7909 * Inform the corresponding mcache(s) that there's a waiter below.
7910 */
7911 static void
7912 mbuf_waiter_inc(mbuf_class_t class, boolean_t comp)
7913 {
7914 mcache_waiter_inc(m_cache(class));
7915 if (comp) {
7916 if (class == MC_CL) {
7917 mcache_waiter_inc(m_cache(MC_MBUF_CL));
7918 } else if (class == MC_BIGCL) {
7919 mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
7920 } else if (class == MC_16KCL) {
7921 mcache_waiter_inc(m_cache(MC_MBUF_16KCL));
7922 } else {
7923 mcache_waiter_inc(m_cache(MC_MBUF_CL));
7924 mcache_waiter_inc(m_cache(MC_MBUF_BIGCL));
7925 }
7926 }
7927 }
7928
7929 /*
7930 * Inform the corresponding mcache(s) that there's no more waiter below.
7931 */
7932 static void
7933 mbuf_waiter_dec(mbuf_class_t class, boolean_t comp)
7934 {
7935 mcache_waiter_dec(m_cache(class));
7936 if (comp) {
7937 if (class == MC_CL) {
7938 mcache_waiter_dec(m_cache(MC_MBUF_CL));
7939 } else if (class == MC_BIGCL) {
7940 mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
7941 } else if (class == MC_16KCL) {
7942 mcache_waiter_dec(m_cache(MC_MBUF_16KCL));
7943 } else {
7944 mcache_waiter_dec(m_cache(MC_MBUF_CL));
7945 mcache_waiter_dec(m_cache(MC_MBUF_BIGCL));
7946 }
7947 }
7948 }
7949
7950 static bool mbuf_watchdog_defunct_active = false;
7951
7952 #endif /* CONFIG_MBUF_MCACHE */
7953
7954 static uint32_t
7955 mbuf_watchdog_socket_space(struct socket *so)
7956 {
7957 uint32_t space = 0;
7958
7959 if (so == NULL) {
7960 return 0;
7961 }
7962
7963 space = so->so_snd.sb_mbcnt + so->so_rcv.sb_mbcnt;
7964
7965 #if INET
7966 if ((SOCK_DOM(so) == PF_INET || SOCK_DOM(so) == PF_INET6) &&
7967 SOCK_PROTO(so) == IPPROTO_TCP) {
7968 space += tcp_reass_qlen_space(so);
7969 }
7970 #endif /* INET */
7971
7972 return space;
7973 }
7974
7975 struct mbuf_watchdog_defunct_args {
7976 struct proc *top_app;
7977 uint32_t top_app_space_used;
7978 bool non_blocking;
7979 };
7980
7981 static bool
7982 proc_fd_trylock(proc_t p)
7983 {
7984 return lck_mtx_try_lock(&p->p_fd.fd_lock);
7985 }
7986
7987 static int
7988 mbuf_watchdog_defunct_iterate(proc_t p, void *arg)
7989 {
7990 struct fileproc *fp = NULL;
7991 struct mbuf_watchdog_defunct_args *args =
7992 (struct mbuf_watchdog_defunct_args *)arg;
7993 uint32_t space_used = 0;
7994
7995 /*
7996 * Non-blocking is only used when dumping the mbuf usage from the watchdog
7997 */
7998 if (args->non_blocking) {
7999 if (!proc_fd_trylock(p)) {
8000 return PROC_RETURNED;
8001 }
8002 } else {
8003 proc_fdlock(p);
8004 }
8005 fdt_foreach(fp, p) {
8006 struct fileglob *fg = fp->fp_glob;
8007 struct socket *so = NULL;
8008
8009 if (FILEGLOB_DTYPE(fg) != DTYPE_SOCKET) {
8010 continue;
8011 }
8012 so = fg_get_data(fg);
8013 /*
8014 * We calculate the space without the socket
8015 * lock because we don't want to be blocked
8016 * by another process that called send() and
8017 * is stuck waiting for mbufs.
8018 *
8019 * These variables are 32-bit so we don't have
8020 * to worry about incomplete reads.
8021 */
8022 space_used += mbuf_watchdog_socket_space(so);
8023 }
8024 proc_fdunlock(p);
8025 if (space_used > args->top_app_space_used) {
8026 if (args->top_app != NULL) {
8027 proc_rele(args->top_app);
8028 }
8029 args->top_app = p;
8030 args->top_app_space_used = space_used;
8031
8032 return PROC_CLAIMED;
8033 } else {
8034 return PROC_RETURNED;
8035 }
8036 }
8037
8038 extern char *proc_name_address(void *p);
8039
8040 static void
8041 mbuf_watchdog_defunct(thread_call_param_t arg0, thread_call_param_t arg1)
8042 {
8043 #pragma unused(arg0, arg1)
8044 struct mbuf_watchdog_defunct_args args = {};
8045 struct fileproc *fp = NULL;
8046
8047 args.non_blocking = false;
8048 proc_iterate(PROC_ALLPROCLIST,
8049 mbuf_watchdog_defunct_iterate, &args, NULL, NULL);
8050
8051 /*
8052 * Defunct all sockets from this app.
8053 */
8054 if (args.top_app != NULL) {
8055 #if CONFIG_MBUF_MCACHE
8056 /* Restart the watchdog count. */
8057 lck_mtx_lock(mbuf_mlock);
8058 microuptime(&mb_wdtstart);
8059 lck_mtx_unlock(mbuf_mlock);
8060 #endif
8061 os_log(OS_LOG_DEFAULT, "%s: defuncting all sockets from %s.%d",
8062 __func__,
8063 proc_name_address(args.top_app),
8064 proc_pid(args.top_app));
8065 proc_fdlock(args.top_app);
8066 fdt_foreach(fp, args.top_app) {
8067 struct fileglob *fg = fp->fp_glob;
8068 struct socket *so = NULL;
8069
8070 if (FILEGLOB_DTYPE(fg) != DTYPE_SOCKET) {
8071 continue;
8072 }
8073 so = (struct socket *)fp_get_data(fp);
8074 if (!socket_try_lock(so)) {
8075 continue;
8076 }
8077 if (sosetdefunct(args.top_app, so,
8078 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL,
8079 TRUE) == 0) {
8080 sodefunct(args.top_app, so,
8081 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL);
8082 }
8083 socket_unlock(so, 0);
8084 }
8085 proc_fdunlock(args.top_app);
8086 proc_rele(args.top_app);
8087 mbstat.m_forcedefunct++;
8088 #if !CONFIG_MBUF_MCACHE
8089 zcache_drain(ZONE_ID_MBUF_CLUSTER_2K);
8090 zcache_drain(ZONE_ID_MBUF_CLUSTER_4K);
8091 zcache_drain(ZONE_ID_MBUF_CLUSTER_16K);
8092 zone_drain(zone_by_id(ZONE_ID_MBUF));
8093 zone_drain(zone_by_id(ZONE_ID_CLUSTER_2K));
8094 zone_drain(zone_by_id(ZONE_ID_CLUSTER_4K));
8095 zone_drain(zone_by_id(ZONE_ID_CLUSTER_16K));
8096 zone_drain(zone_by_id(ZONE_ID_MBUF_REF));
8097 #endif
8098 }
8099 #if CONFIG_MBUF_MCACHE
8100 mbuf_watchdog_defunct_active = false;
8101 #endif
8102 }
8103
8104 #if !CONFIG_MBUF_MCACHE
8105 static LCK_GRP_DECLARE(mbuf_exhausted_grp, "mbuf-exhausted");
8106 static LCK_TICKET_DECLARE(mbuf_exhausted_lock, &mbuf_exhausted_grp);
8107 static uint32_t mbuf_exhausted_mask;
8108
8109 #define MBUF_EXHAUSTED_DRAIN_MASK (\
8110 (1u << MC_MBUF) | \
8111 (1u << MC_CL) | \
8112 (1u << MC_BIGCL) | \
8113 (1u << MC_16KCL))
8114
8115 #define MBUF_EXHAUSTED_DEFUNCT_MASK (\
8116 (1u << MC_MBUF) | \
8117 (1u << MC_MBUF_CL) | \
8118 (1u << MC_MBUF_BIGCL) | \
8119 (1u << MC_MBUF_16KCL))
8120
8121 static void
8122 mbuf_watchdog_drain_composite(thread_call_param_t arg0, thread_call_param_t arg1)
8123 {
8124 #pragma unused(arg0, arg1)
8125 zcache_drain(ZONE_ID_MBUF_CLUSTER_2K);
8126 zcache_drain(ZONE_ID_MBUF_CLUSTER_4K);
8127 zcache_drain(ZONE_ID_MBUF_CLUSTER_16K);
8128 }
8129
8130 static void
8131 mbuf_zone_exhausted_start(uint32_t bit)
8132 {
8133 uint64_t deadline;
8134 uint32_t mask;
8135
8136 mask = mbuf_exhausted_mask;
8137 mbuf_exhausted_mask = mask | bit;
8138
8139 if ((mask & MBUF_EXHAUSTED_DRAIN_MASK) == 0 &&
8140 (bit & MBUF_EXHAUSTED_DRAIN_MASK)) {
8141 clock_interval_to_deadline(MB_WDT_MAXTIME * 1000 / 10,
8142 NSEC_PER_MSEC, &deadline);
8143 thread_call_enter_delayed(mbuf_drain_tcall, deadline);
8144 }
8145
8146 if ((mask & MBUF_EXHAUSTED_DEFUNCT_MASK) == 0 &&
8147 (bit & MBUF_EXHAUSTED_DEFUNCT_MASK)) {
8148 clock_interval_to_deadline(MB_WDT_MAXTIME * 1000 / 2,
8149 NSEC_PER_MSEC, &deadline);
8150 thread_call_enter_delayed(mbuf_defunct_tcall, deadline);
8151 }
8152 }
8153
8154 static void
8155 mbuf_zone_exhausted_end(uint32_t bit)
8156 {
8157 uint32_t mask;
8158
8159 mask = (mbuf_exhausted_mask &= ~bit);
8160
8161 if ((mask & MBUF_EXHAUSTED_DRAIN_MASK) == 0 &&
8162 (bit & MBUF_EXHAUSTED_DRAIN_MASK)) {
8163 thread_call_cancel(mbuf_drain_tcall);
8164 }
8165
8166 if ((mask & MBUF_EXHAUSTED_DEFUNCT_MASK) == 0 &&
8167 (bit & MBUF_EXHAUSTED_DEFUNCT_MASK)) {
8168 thread_call_cancel(mbuf_defunct_tcall);
8169 }
8170 }
8171
8172 static void
8173 mbuf_zone_exhausted(zone_id_t zid, zone_t zone __unused, bool exhausted)
8174 {
8175 uint32_t bit;
8176
8177 if (zid < m_class_to_zid(MBUF_CLASS_MIN) ||
8178 zid > m_class_to_zid(MBUF_CLASS_MAX)) {
8179 return;
8180 }
8181
8182 bit = 1u << m_class_from_zid(zid);
8183
8184 lck_ticket_lock_nopreempt(&mbuf_exhausted_lock, &mbuf_exhausted_grp);
8185
8186 if (exhausted) {
8187 mbuf_zone_exhausted_start(bit);
8188 } else {
8189 mbuf_zone_exhausted_end(bit);
8190 }
8191
8192 lck_ticket_unlock_nopreempt(&mbuf_exhausted_lock);
8193 }
8194 EVENT_REGISTER_HANDLER(ZONE_EXHAUSTED, mbuf_zone_exhausted);
8195 #endif /* !CONFIG_MBUF_MCACHE */
8196
8197 #if CONFIG_MBUF_MCACHE
8198 /*
8199 * Called during slab (blocking and non-blocking) allocation. If there
8200 * is at least one waiter, and the time since the first waiter is blocked
8201 * is greater than the watchdog timeout, panic the system.
8202 */
8203 static void
8204 mbuf_watchdog(void)
8205 {
8206 struct timeval now;
8207 unsigned int since;
8208 static thread_call_t defunct_tcall = NULL;
8209
8210 if (mb_waiters == 0 || !mb_watchdog) {
8211 return;
8212 }
8213
8214 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
8215
8216 microuptime(&now);
8217 since = now.tv_sec - mb_wdtstart.tv_sec;
8218
8219 if (mbuf_watchdog_defunct_active) {
8220 /*
8221 * Don't panic the system while we are trying
8222 * to find sockets to defunct.
8223 */
8224 return;
8225 }
8226 if (since >= MB_WDT_MAXTIME) {
8227 panic_plain("%s: %d waiters stuck for %u secs\n%s", __func__,
8228 mb_waiters, since, mbuf_dump());
8229 /* NOTREACHED */
8230 }
8231 /*
8232 * Check if we are about to panic the system due
8233 * to lack of mbufs and start defuncting sockets
8234 * from processes that use too many sockets.
8235 *
8236 * We're always called with the mbuf_mlock held,
8237 * so that also protects mbuf_watchdog_defunct_active.
8238 */
8239 if (since >= MB_WDT_MAXTIME / 2) {
8240 /*
8241 * Start a thread to defunct sockets
8242 * from apps that are over-using their socket
8243 * buffers.
8244 */
8245 if (defunct_tcall == NULL) {
8246 defunct_tcall =
8247 thread_call_allocate_with_options(mbuf_watchdog_defunct,
8248 NULL,
8249 THREAD_CALL_PRIORITY_KERNEL,
8250 THREAD_CALL_OPTIONS_ONCE);
8251 }
8252 if (defunct_tcall != NULL) {
8253 mbuf_watchdog_defunct_active = true;
8254 thread_call_enter(defunct_tcall);
8255 }
8256 }
8257 }
8258
8259 /*
8260 * Called during blocking allocation. Returns TRUE if one or more objects
8261 * are available at the per-CPU caches layer and that allocation should be
8262 * retried at that level.
8263 */
8264 static boolean_t
8265 mbuf_sleep(mbuf_class_t class, unsigned int num, int wait)
8266 {
8267 boolean_t mcache_retry = FALSE;
8268
8269 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
8270
8271 /* Check if there's anything at the cache layer */
8272 if (mbuf_cached_above(class, wait)) {
8273 mcache_retry = TRUE;
8274 goto done;
8275 }
8276
8277 /* Nothing? Then try hard to get it from somewhere */
8278 m_reclaim(class, num, (wait & MCR_COMP));
8279
8280 /* We tried hard and got something? */
8281 if (m_infree(class) > 0) {
8282 mbstat.m_wait++;
8283 goto done;
8284 } else if (mbuf_cached_above(class, wait)) {
8285 mbstat.m_wait++;
8286 mcache_retry = TRUE;
8287 goto done;
8288 } else if (wait & MCR_TRYHARD) {
8289 mcache_retry = TRUE;
8290 goto done;
8291 }
8292
8293 /*
8294 * There's really nothing for us right now; inform the
8295 * cache(s) that there is a waiter below and go to sleep.
8296 */
8297 mbuf_waiter_inc(class, (wait & MCR_COMP));
8298
8299 VERIFY(!(wait & MCR_NOSLEEP));
8300
8301 /*
8302 * If this is the first waiter, arm the watchdog timer. Otherwise
8303 * check if we need to panic the system due to watchdog timeout.
8304 */
8305 if (mb_waiters == 0) {
8306 microuptime(&mb_wdtstart);
8307 } else {
8308 mbuf_watchdog();
8309 }
8310
8311 mb_waiters++;
8312 m_region_expand(class) += m_total(class) + num;
8313 /* wake up the worker thread */
8314 if (mbuf_worker_ready &&
8315 mbuf_worker_needs_wakeup) {
8316 wakeup((caddr_t)&mbuf_worker_needs_wakeup);
8317 mbuf_worker_needs_wakeup = FALSE;
8318 }
8319 mbwdog_logger("waiting (%d mbufs in class %s)", num, m_cname(class));
8320 (void) msleep(mb_waitchan, mbuf_mlock, (PZERO - 1), m_cname(class), NULL);
8321 mbwdog_logger("woke up (%d mbufs in class %s) ", num, m_cname(class));
8322
8323 /* We are now up; stop getting notified until next round */
8324 mbuf_waiter_dec(class, (wait & MCR_COMP));
8325
8326 /* We waited and got something */
8327 if (m_infree(class) > 0) {
8328 mbstat.m_wait++;
8329 goto done;
8330 } else if (mbuf_cached_above(class, wait)) {
8331 mbstat.m_wait++;
8332 mcache_retry = TRUE;
8333 }
8334 done:
8335 return mcache_retry;
8336 }
8337
8338 __attribute__((noreturn))
8339 static void
8340 mbuf_worker_thread(void)
8341 {
8342 int mbuf_expand;
8343
8344 while (1) {
8345 lck_mtx_lock(mbuf_mlock);
8346 mbwdog_logger("worker thread running");
8347 mbuf_worker_run_cnt++;
8348 mbuf_expand = 0;
8349 /*
8350 * Allocations are based on page size, so if we have depleted
8351 * the reserved spaces, try to free mbufs from the major classes.
8352 */
8353 #if PAGE_SIZE == 4096
8354 uint32_t m_mbclusters = m_total(MC_MBUF) >> NMBPCLSHIFT;
8355 uint32_t m_clusters = m_total(MC_CL);
8356 uint32_t m_bigclusters = m_total(MC_BIGCL) << NCLPBGSHIFT;
8357 uint32_t sumclusters = m_mbclusters + m_clusters + m_bigclusters;
8358 if (sumclusters >= nclusters) {
8359 mbwdog_logger("reclaiming bigcl");
8360 mbuf_drain_locked(TRUE);
8361 m_reclaim(MC_BIGCL, 4, FALSE);
8362 }
8363 #else
8364 uint32_t m_16kclusters = m_total(MC_16KCL);
8365 if (njcl > 0 && (m_16kclusters << NCLPJCLSHIFT) >= njcl) {
8366 mbwdog_logger("reclaiming 16kcl");
8367 mbuf_drain_locked(TRUE);
8368 m_reclaim(MC_16KCL, 4, FALSE);
8369 }
8370 #endif
8371 if (m_region_expand(MC_CL) > 0) {
8372 int n;
8373 mb_expand_cl_cnt++;
8374 /* Adjust to current number of cluster in use */
8375 n = m_region_expand(MC_CL) -
8376 (m_total(MC_CL) - m_infree(MC_CL));
8377 if ((n + m_total(MC_CL)) > m_maxlimit(MC_CL)) {
8378 n = m_maxlimit(MC_CL) - m_total(MC_CL);
8379 }
8380 if (n > 0) {
8381 mb_expand_cl_total += n;
8382 }
8383 m_region_expand(MC_CL) = 0;
8384
8385 if (n > 0) {
8386 mbwdog_logger("expanding MC_CL by %d", n);
8387 freelist_populate(MC_CL, n, M_WAIT);
8388 }
8389 }
8390 if (m_region_expand(MC_BIGCL) > 0) {
8391 int n;
8392 mb_expand_bigcl_cnt++;
8393 /* Adjust to current number of 4 KB cluster in use */
8394 n = m_region_expand(MC_BIGCL) -
8395 (m_total(MC_BIGCL) - m_infree(MC_BIGCL));
8396 if ((n + m_total(MC_BIGCL)) > m_maxlimit(MC_BIGCL)) {
8397 n = m_maxlimit(MC_BIGCL) - m_total(MC_BIGCL);
8398 }
8399 if (n > 0) {
8400 mb_expand_bigcl_total += n;
8401 }
8402 m_region_expand(MC_BIGCL) = 0;
8403
8404 if (n > 0) {
8405 mbwdog_logger("expanding MC_BIGCL by %d", n);
8406 freelist_populate(MC_BIGCL, n, M_WAIT);
8407 }
8408 }
8409 if (m_region_expand(MC_16KCL) > 0) {
8410 int n;
8411 mb_expand_16kcl_cnt++;
8412 /* Adjust to current number of 16 KB cluster in use */
8413 n = m_region_expand(MC_16KCL) -
8414 (m_total(MC_16KCL) - m_infree(MC_16KCL));
8415 if ((n + m_total(MC_16KCL)) > m_maxlimit(MC_16KCL)) {
8416 n = m_maxlimit(MC_16KCL) - m_total(MC_16KCL);
8417 }
8418 if (n > 0) {
8419 mb_expand_16kcl_total += n;
8420 }
8421 m_region_expand(MC_16KCL) = 0;
8422
8423 if (n > 0) {
8424 mbwdog_logger("expanding MC_16KCL by %d", n);
8425 (void) freelist_populate(MC_16KCL, n, M_WAIT);
8426 }
8427 }
8428
8429 /*
8430 * Because we can run out of memory before filling the mbuf
8431 * map, we should not allocate more clusters than they are
8432 * mbufs -- otherwise we could have a large number of useless
8433 * clusters allocated.
8434 */
8435 mbwdog_logger("totals: MC_MBUF %d MC_BIGCL %d MC_CL %d MC_16KCL %d",
8436 m_total(MC_MBUF), m_total(MC_BIGCL), m_total(MC_CL),
8437 m_total(MC_16KCL));
8438 uint32_t total_mbufs = m_total(MC_MBUF);
8439 uint32_t total_clusters = m_total(MC_BIGCL) + m_total(MC_CL) +
8440 m_total(MC_16KCL);
8441 if (total_mbufs < total_clusters) {
8442 mbwdog_logger("expanding MC_MBUF by %d",
8443 total_clusters - total_mbufs);
8444 }
8445 while (total_mbufs < total_clusters) {
8446 mb_expand_cnt++;
8447 if (freelist_populate(MC_MBUF, 1, M_WAIT) == 0) {
8448 break;
8449 }
8450 total_mbufs = m_total(MC_MBUF);
8451 total_clusters = m_total(MC_BIGCL) + m_total(MC_CL) +
8452 m_total(MC_16KCL);
8453 }
8454
8455 mbuf_worker_needs_wakeup = TRUE;
8456 /*
8457 * If there's a deadlock and we're not sending / receiving
8458 * packets, net_uptime() won't be updated. Update it here
8459 * so we are sure it's correct.
8460 */
8461 net_update_uptime();
8462 mbuf_worker_last_runtime = net_uptime();
8463 assert_wait((caddr_t)&mbuf_worker_needs_wakeup,
8464 THREAD_UNINT);
8465 mbwdog_logger("worker thread sleeping");
8466 lck_mtx_unlock(mbuf_mlock);
8467 (void) thread_block((thread_continue_t)mbuf_worker_thread);
8468 }
8469 }
8470
8471 __attribute__((noreturn))
8472 static void
8473 mbuf_worker_thread_init(void)
8474 {
8475 mbuf_worker_ready++;
8476 mbuf_worker_thread();
8477 }
8478
8479 static mcl_slab_t *
8480 slab_get(void *buf)
8481 {
8482 mcl_slabg_t *slg;
8483 unsigned int ix, k;
8484
8485 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
8486
8487 VERIFY(MBUF_IN_MAP(buf));
8488 ix = ((unsigned char *)buf - mbutl) >> MBSHIFT;
8489 VERIFY(ix < maxslabgrp);
8490
8491 if ((slg = slabstbl[ix]) == NULL) {
8492 /*
8493 * In the current implementation, we never shrink the slabs
8494 * table; if we attempt to reallocate a cluster group when
8495 * it's already allocated, panic since this is a sign of a
8496 * memory corruption (slabstbl[ix] got nullified).
8497 */
8498 ++slabgrp;
8499 VERIFY(ix < slabgrp);
8500 /*
8501 * Slabs expansion can only be done single threaded; when
8502 * we get here, it must be as a result of m_clalloc() which
8503 * is serialized and therefore mb_clalloc_busy must be set.
8504 */
8505 VERIFY(mb_clalloc_busy);
8506 lck_mtx_unlock(mbuf_mlock);
8507
8508 /* This is a new buffer; create the slabs group for it */
8509 slg = zalloc_permanent_type(mcl_slabg_t);
8510 slg->slg_slab = zalloc_permanent(sizeof(mcl_slab_t) * NSLABSPMB,
8511 ZALIGN(mcl_slab_t));
8512
8513 lck_mtx_lock(mbuf_mlock);
8514 /*
8515 * No other thread could have gone into m_clalloc() after
8516 * we dropped the lock above, so verify that it's true.
8517 */
8518 VERIFY(mb_clalloc_busy);
8519
8520 slabstbl[ix] = slg;
8521
8522 /* Chain each slab in the group to its forward neighbor */
8523 for (k = 1; k < NSLABSPMB; k++) {
8524 slg->slg_slab[k - 1].sl_next = &slg->slg_slab[k];
8525 }
8526 VERIFY(slg->slg_slab[NSLABSPMB - 1].sl_next == NULL);
8527
8528 /* And chain the last slab in the previous group to this */
8529 if (ix > 0) {
8530 VERIFY(slabstbl[ix - 1]->
8531 slg_slab[NSLABSPMB - 1].sl_next == NULL);
8532 slabstbl[ix - 1]->slg_slab[NSLABSPMB - 1].sl_next =
8533 &slg->slg_slab[0];
8534 }
8535 }
8536
8537 ix = MTOPG(buf) % NSLABSPMB;
8538 VERIFY(ix < NSLABSPMB);
8539
8540 return &slg->slg_slab[ix];
8541 }
8542
8543 static void
8544 slab_init(mcl_slab_t *sp, mbuf_class_t class, u_int32_t flags,
8545 void *base, void *head, unsigned int len, int refcnt, int chunks)
8546 {
8547 sp->sl_class = class;
8548 sp->sl_flags = flags;
8549 sp->sl_base = base;
8550 sp->sl_head = head;
8551 sp->sl_len = len;
8552 sp->sl_refcnt = refcnt;
8553 sp->sl_chunks = chunks;
8554 slab_detach(sp);
8555 }
8556
8557 static void
8558 slab_insert(mcl_slab_t *sp, mbuf_class_t class)
8559 {
8560 VERIFY(slab_is_detached(sp));
8561 m_slab_cnt(class)++;
8562 TAILQ_INSERT_TAIL(&m_slablist(class), sp, sl_link);
8563 sp->sl_flags &= ~SLF_DETACHED;
8564
8565 /*
8566 * If a buffer spans multiple contiguous pages then mark them as
8567 * detached too
8568 */
8569 if (class == MC_16KCL) {
8570 int k;
8571 for (k = 1; k < NSLABSP16KB; k++) {
8572 sp = sp->sl_next;
8573 /* Next slab must already be present */
8574 VERIFY(sp != NULL && slab_is_detached(sp));
8575 sp->sl_flags &= ~SLF_DETACHED;
8576 }
8577 }
8578 }
8579
8580 static void
8581 slab_remove(mcl_slab_t *sp, mbuf_class_t class)
8582 {
8583 int k;
8584 VERIFY(!slab_is_detached(sp));
8585 VERIFY(m_slab_cnt(class) > 0);
8586 m_slab_cnt(class)--;
8587 TAILQ_REMOVE(&m_slablist(class), sp, sl_link);
8588 slab_detach(sp);
8589 if (class == MC_16KCL) {
8590 for (k = 1; k < NSLABSP16KB; k++) {
8591 sp = sp->sl_next;
8592 /* Next slab must already be present */
8593 VERIFY(sp != NULL);
8594 VERIFY(!slab_is_detached(sp));
8595 slab_detach(sp);
8596 }
8597 }
8598 }
8599
8600 static boolean_t
8601 slab_inrange(mcl_slab_t *sp, void *buf)
8602 {
8603 return (uintptr_t)buf >= (uintptr_t)sp->sl_base &&
8604 (uintptr_t)buf < ((uintptr_t)sp->sl_base + sp->sl_len);
8605 }
8606
8607 #undef panic
8608
8609 static void
8610 slab_nextptr_panic(mcl_slab_t *sp, void *addr)
8611 {
8612 int i;
8613 unsigned int chunk_len = sp->sl_len / sp->sl_chunks;
8614 uintptr_t buf = (uintptr_t)sp->sl_base;
8615
8616 for (i = 0; i < sp->sl_chunks; i++, buf += chunk_len) {
8617 void *next = ((mcache_obj_t *)buf)->obj_next;
8618 if (next != addr) {
8619 continue;
8620 }
8621 if (!mclverify) {
8622 if (next != NULL && !MBUF_IN_MAP(next)) {
8623 mcache_t *cp = m_cache(sp->sl_class);
8624 panic("%s: %s buffer %p in slab %p modified "
8625 "after free at offset 0: %p out of range "
8626 "[%p-%p)\n", __func__, cp->mc_name,
8627 (void *)buf, sp, next, mbutl, embutl);
8628 /* NOTREACHED */
8629 }
8630 } else {
8631 mcache_audit_t *mca = mcl_audit_buf2mca(sp->sl_class,
8632 (mcache_obj_t *)buf);
8633 mcl_audit_verify_nextptr(next, mca);
8634 }
8635 }
8636 }
8637
8638 static void
8639 slab_detach(mcl_slab_t *sp)
8640 {
8641 sp->sl_link.tqe_next = (mcl_slab_t *)-1;
8642 sp->sl_link.tqe_prev = (mcl_slab_t **)-1;
8643 sp->sl_flags |= SLF_DETACHED;
8644 }
8645
8646 static boolean_t
8647 slab_is_detached(mcl_slab_t *sp)
8648 {
8649 return (intptr_t)sp->sl_link.tqe_next == -1 &&
8650 (intptr_t)sp->sl_link.tqe_prev == -1 &&
8651 (sp->sl_flags & SLF_DETACHED);
8652 }
8653
8654 static void
8655 mcl_audit_init(void *buf, mcache_audit_t **mca_list,
8656 mcache_obj_t **con_list, size_t con_size, unsigned int num)
8657 {
8658 mcache_audit_t *mca, *mca_tail;
8659 mcache_obj_t *con = NULL;
8660 boolean_t save_contents = (con_list != NULL);
8661 unsigned int i, ix;
8662
8663 ASSERT(num <= NMBPG);
8664 ASSERT(con_list == NULL || con_size != 0);
8665
8666 ix = MTOPG(buf);
8667 VERIFY(ix < maxclaudit);
8668
8669 /* Make sure we haven't been here before */
8670 for (i = 0; i < num; i++) {
8671 VERIFY(mclaudit[ix].cl_audit[i] == NULL);
8672 }
8673
8674 mca = mca_tail = *mca_list;
8675 if (save_contents) {
8676 con = *con_list;
8677 }
8678
8679 for (i = 0; i < num; i++) {
8680 mcache_audit_t *next;
8681
8682 next = mca->mca_next;
8683 bzero(mca, sizeof(*mca));
8684 mca->mca_next = next;
8685 mclaudit[ix].cl_audit[i] = mca;
8686
8687 /* Attach the contents buffer if requested */
8688 if (save_contents) {
8689 mcl_saved_contents_t *msc =
8690 (mcl_saved_contents_t *)(void *)con;
8691
8692 VERIFY(msc != NULL);
8693 VERIFY(IS_P2ALIGNED(msc, sizeof(u_int64_t)));
8694 VERIFY(con_size == sizeof(*msc));
8695 mca->mca_contents_size = con_size;
8696 mca->mca_contents = msc;
8697 con = con->obj_next;
8698 bzero(mca->mca_contents, mca->mca_contents_size);
8699 }
8700
8701 mca_tail = mca;
8702 mca = mca->mca_next;
8703 }
8704
8705 if (save_contents) {
8706 *con_list = con;
8707 }
8708
8709 *mca_list = mca_tail->mca_next;
8710 mca_tail->mca_next = NULL;
8711 }
8712
8713 static void
8714 mcl_audit_free(void *buf, unsigned int num)
8715 {
8716 unsigned int i, ix;
8717 mcache_audit_t *mca, *mca_list;
8718
8719 ix = MTOPG(buf);
8720 VERIFY(ix < maxclaudit);
8721
8722 if (mclaudit[ix].cl_audit[0] != NULL) {
8723 mca_list = mclaudit[ix].cl_audit[0];
8724 for (i = 0; i < num; i++) {
8725 mca = mclaudit[ix].cl_audit[i];
8726 mclaudit[ix].cl_audit[i] = NULL;
8727 if (mca->mca_contents) {
8728 mcache_free(mcl_audit_con_cache,
8729 mca->mca_contents);
8730 }
8731 }
8732 mcache_free_ext(mcache_audit_cache,
8733 (mcache_obj_t *)mca_list);
8734 }
8735 }
8736
8737 /*
8738 * Given an address of a buffer (mbuf/2KB/4KB/16KB), return
8739 * the corresponding audit structure for that buffer.
8740 */
8741 static mcache_audit_t *
8742 mcl_audit_buf2mca(mbuf_class_t class, mcache_obj_t *mobj)
8743 {
8744 mcache_audit_t *mca = NULL;
8745 int ix = MTOPG(mobj), m_idx = 0;
8746 unsigned char *page_addr;
8747
8748 VERIFY(ix < maxclaudit);
8749 VERIFY(IS_P2ALIGNED(mobj, MIN(m_maxsize(class), PAGE_SIZE)));
8750
8751 page_addr = PGTOM(ix);
8752
8753 switch (class) {
8754 case MC_MBUF:
8755 /*
8756 * For the mbuf case, find the index of the page
8757 * used by the mbuf and use that index to locate the
8758 * base address of the page. Then find out the
8759 * mbuf index relative to the page base and use
8760 * it to locate the audit structure.
8761 */
8762 m_idx = MBPAGEIDX(page_addr, mobj);
8763 VERIFY(m_idx < (int)NMBPG);
8764 mca = mclaudit[ix].cl_audit[m_idx];
8765 break;
8766
8767 case MC_CL:
8768 /*
8769 * Same thing as above, but for 2KB clusters in a page.
8770 */
8771 m_idx = CLPAGEIDX(page_addr, mobj);
8772 VERIFY(m_idx < (int)NCLPG);
8773 mca = mclaudit[ix].cl_audit[m_idx];
8774 break;
8775
8776 case MC_BIGCL:
8777 m_idx = BCLPAGEIDX(page_addr, mobj);
8778 VERIFY(m_idx < (int)NBCLPG);
8779 mca = mclaudit[ix].cl_audit[m_idx];
8780 break;
8781 case MC_16KCL:
8782 /*
8783 * Same as above, but only return the first element.
8784 */
8785 mca = mclaudit[ix].cl_audit[0];
8786 break;
8787
8788 default:
8789 VERIFY(0);
8790 /* NOTREACHED */
8791 }
8792
8793 return mca;
8794 }
8795
8796 static void
8797 mcl_audit_mbuf(mcache_audit_t *mca, void *addr, boolean_t composite,
8798 boolean_t alloc)
8799 {
8800 struct mbuf *m = addr;
8801 mcache_obj_t *next = ((mcache_obj_t *)m)->obj_next;
8802
8803 VERIFY(mca->mca_contents != NULL &&
8804 mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
8805
8806 if (mclverify) {
8807 mcl_audit_verify_nextptr(next, mca);
8808 }
8809
8810 if (!alloc) {
8811 /* Save constructed mbuf fields */
8812 mcl_audit_save_mbuf(m, mca);
8813 if (mclverify) {
8814 mcache_set_pattern(MCACHE_FREE_PATTERN, m,
8815 m_maxsize(MC_MBUF));
8816 }
8817 ((mcache_obj_t *)m)->obj_next = next;
8818 return;
8819 }
8820
8821 /* Check if the buffer has been corrupted while in freelist */
8822 if (mclverify) {
8823 mcache_audit_free_verify_set(mca, addr, 0, m_maxsize(MC_MBUF));
8824 }
8825 /* Restore constructed mbuf fields */
8826 mcl_audit_restore_mbuf(m, mca, composite);
8827 }
8828
8829 static void
8830 mcl_audit_restore_mbuf(struct mbuf *m, mcache_audit_t *mca, boolean_t composite)
8831 {
8832 struct mbuf *ms = MCA_SAVED_MBUF_PTR(mca);
8833
8834 if (composite) {
8835 struct mbuf *next = m->m_next;
8836 VERIFY(ms->m_flags == M_EXT && m_get_rfa(ms) != NULL &&
8837 MBUF_IS_COMPOSITE(ms));
8838 VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
8839 /*
8840 * We could have hand-picked the mbuf fields and restore
8841 * them individually, but that will be a maintenance
8842 * headache. Instead, restore everything that was saved;
8843 * the mbuf layer will recheck and reinitialize anyway.
8844 */
8845 bcopy(ms, m, MCA_SAVED_MBUF_SIZE);
8846 m->m_next = next;
8847 } else {
8848 /*
8849 * For a regular mbuf (no cluster attached) there's nothing
8850 * to restore other than the type field, which is expected
8851 * to be MT_FREE.
8852 */
8853 m->m_type = ms->m_type;
8854 }
8855 _MCHECK(m);
8856 }
8857
8858 static void
8859 mcl_audit_save_mbuf(struct mbuf *m, mcache_audit_t *mca)
8860 {
8861 VERIFY(mca->mca_contents_size == AUDIT_CONTENTS_SIZE);
8862 _MCHECK(m);
8863 bcopy(m, MCA_SAVED_MBUF_PTR(mca), MCA_SAVED_MBUF_SIZE);
8864 }
8865
8866 static void
8867 mcl_audit_cluster(mcache_audit_t *mca, void *addr, size_t size, boolean_t alloc,
8868 boolean_t save_next)
8869 {
8870 mcache_obj_t *next = ((mcache_obj_t *)addr)->obj_next;
8871
8872 if (!alloc) {
8873 if (mclverify) {
8874 mcache_set_pattern(MCACHE_FREE_PATTERN, addr, size);
8875 }
8876 if (save_next) {
8877 mcl_audit_verify_nextptr(next, mca);
8878 ((mcache_obj_t *)addr)->obj_next = next;
8879 }
8880 } else if (mclverify) {
8881 /* Check if the buffer has been corrupted while in freelist */
8882 mcl_audit_verify_nextptr(next, mca);
8883 mcache_audit_free_verify_set(mca, addr, 0, size);
8884 }
8885 }
8886
8887 static void
8888 mcl_audit_scratch(mcache_audit_t *mca)
8889 {
8890 void *stack[MCACHE_STACK_DEPTH + 1];
8891 mcl_scratch_audit_t *msa;
8892 struct timeval now;
8893
8894 VERIFY(mca->mca_contents != NULL);
8895 msa = MCA_SAVED_SCRATCH_PTR(mca);
8896
8897 msa->msa_pthread = msa->msa_thread;
8898 msa->msa_thread = current_thread();
8899 bcopy(msa->msa_stack, msa->msa_pstack, sizeof(msa->msa_pstack));
8900 msa->msa_pdepth = msa->msa_depth;
8901 bzero(stack, sizeof(stack));
8902 msa->msa_depth = OSBacktrace(stack, MCACHE_STACK_DEPTH + 1) - 1;
8903 bcopy(&stack[1], msa->msa_stack, sizeof(msa->msa_stack));
8904
8905 msa->msa_ptstamp = msa->msa_tstamp;
8906 microuptime(&now);
8907 /* tstamp is in ms relative to base_ts */
8908 msa->msa_tstamp = ((now.tv_usec - mb_start.tv_usec) / 1000);
8909 if ((now.tv_sec - mb_start.tv_sec) > 0) {
8910 msa->msa_tstamp += ((now.tv_sec - mb_start.tv_sec) * 1000);
8911 }
8912 }
8913
8914 __abortlike
8915 static void
8916 mcl_audit_mcheck_panic(struct mbuf *m)
8917 {
8918 char buf[DUMP_MCA_BUF_SIZE];
8919 mcache_audit_t *mca;
8920
8921 MRANGE(m);
8922 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
8923
8924 panic("mcl_audit: freed mbuf %p with type 0x%x (instead of 0x%x)\n%s",
8925 m, (u_int16_t)m->m_type, MT_FREE, mcache_dump_mca(buf, mca));
8926 /* NOTREACHED */
8927 }
8928
8929 __abortlike
8930 static void
8931 mcl_audit_verify_nextptr_panic(void *next, mcache_audit_t *mca)
8932 {
8933 char buf[DUMP_MCA_BUF_SIZE];
8934 panic("mcl_audit: buffer %p modified after free at offset 0: "
8935 "%p out of range [%p-%p)\n%s\n",
8936 mca->mca_addr, next, mbutl, embutl, mcache_dump_mca(buf, mca));
8937 /* NOTREACHED */
8938 }
8939
8940 static void
8941 mcl_audit_verify_nextptr(void *next, mcache_audit_t *mca)
8942 {
8943 if (next != NULL && !MBUF_IN_MAP(next) &&
8944 (next != (void *)MCACHE_FREE_PATTERN || !mclverify)) {
8945 mcl_audit_verify_nextptr_panic(next, mca);
8946 }
8947 }
8948
8949 static uintptr_t
8950 hash_mix(uintptr_t x)
8951 {
8952 #ifndef __LP64__
8953 x += ~(x << 15);
8954 x ^= (x >> 10);
8955 x += (x << 3);
8956 x ^= (x >> 6);
8957 x += ~(x << 11);
8958 x ^= (x >> 16);
8959 #else
8960 x += ~(x << 32);
8961 x ^= (x >> 22);
8962 x += ~(x << 13);
8963 x ^= (x >> 8);
8964 x += (x << 3);
8965 x ^= (x >> 15);
8966 x += ~(x << 27);
8967 x ^= (x >> 31);
8968 #endif
8969 return x;
8970 }
8971
8972 static uint32_t
8973 hashbacktrace(uintptr_t* bt, uint32_t depth, uint32_t max_size)
8974 {
8975 uintptr_t hash = 0;
8976 uintptr_t mask = max_size - 1;
8977
8978 while (depth) {
8979 hash += bt[--depth];
8980 }
8981
8982 hash = hash_mix(hash) & mask;
8983
8984 assert(hash < max_size);
8985
8986 return (uint32_t) hash;
8987 }
8988
8989 static uint32_t
8990 hashaddr(uintptr_t pt, uint32_t max_size)
8991 {
8992 uintptr_t hash = 0;
8993 uintptr_t mask = max_size - 1;
8994
8995 hash = hash_mix(pt) & mask;
8996
8997 assert(hash < max_size);
8998
8999 return (uint32_t) hash;
9000 }
9001
9002 /* This function turns on mbuf leak detection */
9003 static void
9004 mleak_activate(void)
9005 {
9006 mleak_table.mleak_sample_factor = MLEAK_SAMPLE_FACTOR;
9007 PE_parse_boot_argn("mleak_sample_factor",
9008 &mleak_table.mleak_sample_factor,
9009 sizeof(mleak_table.mleak_sample_factor));
9010
9011 if (mleak_table.mleak_sample_factor == 0) {
9012 mclfindleak = 0;
9013 }
9014
9015 if (mclfindleak == 0) {
9016 return;
9017 }
9018
9019 vm_size_t alloc_size =
9020 mleak_alloc_buckets * sizeof(struct mallocation);
9021 vm_size_t trace_size = mleak_trace_buckets * sizeof(struct mtrace);
9022
9023 mleak_allocations = zalloc_permanent(alloc_size, ZALIGN(struct mallocation));
9024 mleak_traces = zalloc_permanent(trace_size, ZALIGN(struct mtrace));
9025 mleak_stat = zalloc_permanent(MLEAK_STAT_SIZE(MLEAK_NUM_TRACES),
9026 ZALIGN(mleak_stat_t));
9027
9028 mleak_stat->ml_cnt = MLEAK_NUM_TRACES;
9029 #ifdef __LP64__
9030 mleak_stat->ml_isaddr64 = 1;
9031 #endif /* __LP64__ */
9032 }
9033
9034 static void
9035 mleak_logger(u_int32_t num, mcache_obj_t *addr, boolean_t alloc)
9036 {
9037 int temp;
9038
9039 if (mclfindleak == 0) {
9040 return;
9041 }
9042
9043 if (!alloc) {
9044 return mleak_free(addr);
9045 }
9046
9047 temp = os_atomic_inc_orig(&mleak_table.mleak_capture, relaxed);
9048
9049 if ((temp % mleak_table.mleak_sample_factor) == 0 && addr != NULL) {
9050 uintptr_t bt[MLEAK_STACK_DEPTH];
9051 unsigned int logged = backtrace(bt, MLEAK_STACK_DEPTH, NULL, NULL);
9052 mleak_log(bt, addr, logged, num);
9053 }
9054 }
9055
9056 /*
9057 * This function records the allocation in the mleak_allocations table
9058 * and the backtrace in the mleak_traces table; if allocation slot is in use,
9059 * replace old allocation with new one if the trace slot is in use, return
9060 * (or increment refcount if same trace).
9061 */
9062 static boolean_t
9063 mleak_log(uintptr_t *bt, mcache_obj_t *addr, uint32_t depth, int num)
9064 {
9065 struct mallocation *allocation;
9066 struct mtrace *trace;
9067 uint32_t trace_index;
9068
9069 /* Quit if someone else modifying the tables */
9070 if (!lck_mtx_try_lock_spin(mleak_lock)) {
9071 mleak_table.total_conflicts++;
9072 return FALSE;
9073 }
9074
9075 allocation = &mleak_allocations[hashaddr((uintptr_t)addr,
9076 mleak_alloc_buckets)];
9077 trace_index = hashbacktrace(bt, depth, mleak_trace_buckets);
9078 trace = &mleak_traces[trace_index];
9079
9080 VERIFY(allocation <= &mleak_allocations[mleak_alloc_buckets - 1]);
9081 VERIFY(trace <= &mleak_traces[mleak_trace_buckets - 1]);
9082
9083 allocation->hitcount++;
9084 trace->hitcount++;
9085
9086 /*
9087 * If the allocation bucket we want is occupied
9088 * and the occupier has the same trace, just bail.
9089 */
9090 if (allocation->element != NULL &&
9091 trace_index == allocation->trace_index) {
9092 mleak_table.alloc_collisions++;
9093 lck_mtx_unlock(mleak_lock);
9094 return TRUE;
9095 }
9096
9097 /*
9098 * Store the backtrace in the traces array;
9099 * Size of zero = trace bucket is free.
9100 */
9101 if (trace->allocs > 0 &&
9102 bcmp(trace->addr, bt, (depth * sizeof(uintptr_t))) != 0) {
9103 /* Different, unique trace, but the same hash! Bail out. */
9104 trace->collisions++;
9105 mleak_table.trace_collisions++;
9106 lck_mtx_unlock(mleak_lock);
9107 return TRUE;
9108 } else if (trace->allocs > 0) {
9109 /* Same trace, already added, so increment refcount */
9110 trace->allocs++;
9111 } else {
9112 /* Found an unused trace bucket, so record the trace here */
9113 if (trace->depth != 0) {
9114 /* this slot previously used but not currently in use */
9115 mleak_table.trace_overwrites++;
9116 }
9117 mleak_table.trace_recorded++;
9118 trace->allocs = 1;
9119 memcpy(trace->addr, bt, (depth * sizeof(uintptr_t)));
9120 trace->depth = depth;
9121 trace->collisions = 0;
9122 }
9123
9124 /* Step 2: Store the allocation record in the allocations array */
9125 if (allocation->element != NULL) {
9126 /*
9127 * Replace an existing allocation. No need to preserve
9128 * because only a subset of the allocations are being
9129 * recorded anyway.
9130 */
9131 mleak_table.alloc_collisions++;
9132 } else if (allocation->trace_index != 0) {
9133 mleak_table.alloc_overwrites++;
9134 }
9135 allocation->element = addr;
9136 allocation->trace_index = trace_index;
9137 allocation->count = num;
9138 mleak_table.alloc_recorded++;
9139 mleak_table.outstanding_allocs++;
9140
9141 lck_mtx_unlock(mleak_lock);
9142 return TRUE;
9143 }
9144
9145 static void
9146 mleak_free(mcache_obj_t *addr)
9147 {
9148 while (addr != NULL) {
9149 struct mallocation *allocation = &mleak_allocations
9150 [hashaddr((uintptr_t)addr, mleak_alloc_buckets)];
9151
9152 if (allocation->element == addr &&
9153 allocation->trace_index < mleak_trace_buckets) {
9154 lck_mtx_lock_spin(mleak_lock);
9155 if (allocation->element == addr &&
9156 allocation->trace_index < mleak_trace_buckets) {
9157 struct mtrace *trace;
9158 trace = &mleak_traces[allocation->trace_index];
9159 /* allocs = 0 means trace bucket is unused */
9160 if (trace->allocs > 0) {
9161 trace->allocs--;
9162 }
9163 if (trace->allocs == 0) {
9164 trace->depth = 0;
9165 }
9166 /* NULL element means alloc bucket is unused */
9167 allocation->element = NULL;
9168 mleak_table.outstanding_allocs--;
9169 }
9170 lck_mtx_unlock(mleak_lock);
9171 }
9172 addr = addr->obj_next;
9173 }
9174 }
9175
9176 static void
9177 mleak_sort_traces()
9178 {
9179 int i, j, k;
9180 struct mtrace *swap;
9181
9182 for (i = 0; i < MLEAK_NUM_TRACES; i++) {
9183 mleak_top_trace[i] = NULL;
9184 }
9185
9186 for (i = 0, j = 0; j < MLEAK_NUM_TRACES && i < mleak_trace_buckets; i++) {
9187 if (mleak_traces[i].allocs <= 0) {
9188 continue;
9189 }
9190
9191 mleak_top_trace[j] = &mleak_traces[i];
9192 for (k = j; k > 0; k--) {
9193 if (mleak_top_trace[k]->allocs <=
9194 mleak_top_trace[k - 1]->allocs) {
9195 break;
9196 }
9197
9198 swap = mleak_top_trace[k - 1];
9199 mleak_top_trace[k - 1] = mleak_top_trace[k];
9200 mleak_top_trace[k] = swap;
9201 }
9202 j++;
9203 }
9204
9205 j--;
9206 for (; i < mleak_trace_buckets; i++) {
9207 if (mleak_traces[i].allocs <= mleak_top_trace[j]->allocs) {
9208 continue;
9209 }
9210
9211 mleak_top_trace[j] = &mleak_traces[i];
9212
9213 for (k = j; k > 0; k--) {
9214 if (mleak_top_trace[k]->allocs <=
9215 mleak_top_trace[k - 1]->allocs) {
9216 break;
9217 }
9218
9219 swap = mleak_top_trace[k - 1];
9220 mleak_top_trace[k - 1] = mleak_top_trace[k];
9221 mleak_top_trace[k] = swap;
9222 }
9223 }
9224 }
9225
9226 static void
9227 mleak_update_stats()
9228 {
9229 mleak_trace_stat_t *mltr;
9230 int i;
9231
9232 VERIFY(mleak_stat != NULL);
9233 #ifdef __LP64__
9234 VERIFY(mleak_stat->ml_isaddr64);
9235 #else
9236 VERIFY(!mleak_stat->ml_isaddr64);
9237 #endif /* !__LP64__ */
9238 VERIFY(mleak_stat->ml_cnt == MLEAK_NUM_TRACES);
9239
9240 mleak_sort_traces();
9241
9242 mltr = &mleak_stat->ml_trace[0];
9243 bzero(mltr, sizeof(*mltr) * MLEAK_NUM_TRACES);
9244 for (i = 0; i < MLEAK_NUM_TRACES; i++) {
9245 int j;
9246
9247 if (mleak_top_trace[i] == NULL ||
9248 mleak_top_trace[i]->allocs == 0) {
9249 continue;
9250 }
9251
9252 mltr->mltr_collisions = mleak_top_trace[i]->collisions;
9253 mltr->mltr_hitcount = mleak_top_trace[i]->hitcount;
9254 mltr->mltr_allocs = mleak_top_trace[i]->allocs;
9255 mltr->mltr_depth = mleak_top_trace[i]->depth;
9256
9257 VERIFY(mltr->mltr_depth <= MLEAK_STACK_DEPTH);
9258 for (j = 0; j < mltr->mltr_depth; j++) {
9259 mltr->mltr_addr[j] = mleak_top_trace[i]->addr[j];
9260 }
9261
9262 mltr++;
9263 }
9264 }
9265
9266 static struct mbtypes {
9267 int mt_type;
9268 const char *mt_name;
9269 } mbtypes[] = {
9270 { MT_DATA, "data" },
9271 { MT_OOBDATA, "oob data" },
9272 { MT_CONTROL, "ancillary data" },
9273 { MT_HEADER, "packet headers" },
9274 { MT_SOCKET, "socket structures" },
9275 { MT_PCB, "protocol control blocks" },
9276 { MT_RTABLE, "routing table entries" },
9277 { MT_HTABLE, "IMP host table entries" },
9278 { MT_ATABLE, "address resolution tables" },
9279 { MT_FTABLE, "fragment reassembly queue headers" },
9280 { MT_SONAME, "socket names and addresses" },
9281 { MT_SOOPTS, "socket options" },
9282 { MT_RIGHTS, "access rights" },
9283 { MT_IFADDR, "interface addresses" },
9284 { MT_TAG, "packet tags" },
9285 { 0, NULL }
9286 };
9287
9288 #define MBUF_DUMP_BUF_CHK() { \
9289 clen -= k; \
9290 if (clen < 1) \
9291 goto done; \
9292 c += k; \
9293 }
9294
9295 static char *
9296 mbuf_dump(void)
9297 {
9298 unsigned long totmem = 0, totfree = 0, totmbufs, totused, totpct,
9299 totreturned = 0;
9300 u_int32_t m_mbufs = 0, m_clfree = 0, m_bigclfree = 0;
9301 u_int32_t m_mbufclfree = 0, m_mbufbigclfree = 0;
9302 u_int32_t m_16kclusters = 0, m_16kclfree = 0, m_mbuf16kclfree = 0;
9303 int nmbtypes = sizeof(mbstat.m_mtypes) / sizeof(short);
9304 uint8_t seen[256];
9305 struct mbtypes *mp;
9306 mb_class_stat_t *sp;
9307 mleak_trace_stat_t *mltr;
9308 char *c = mbuf_dump_buf;
9309 int i, j, k, clen = MBUF_DUMP_BUF_SIZE;
9310 struct mbuf_watchdog_defunct_args args = {};
9311
9312 mbuf_dump_buf[0] = '\0';
9313
9314 /* synchronize all statistics in the mbuf table */
9315 mbuf_stat_sync();
9316 mbuf_mtypes_sync(TRUE);
9317
9318 sp = &mb_stat->mbs_class[0];
9319 for (i = 0; i < mb_stat->mbs_cnt; i++, sp++) {
9320 u_int32_t mem;
9321
9322 if (m_class(i) == MC_MBUF) {
9323 m_mbufs = sp->mbcl_active;
9324 } else if (m_class(i) == MC_CL) {
9325 m_clfree = sp->mbcl_total - sp->mbcl_active;
9326 } else if (m_class(i) == MC_BIGCL) {
9327 m_bigclfree = sp->mbcl_total - sp->mbcl_active;
9328 } else if (njcl > 0 && m_class(i) == MC_16KCL) {
9329 m_16kclfree = sp->mbcl_total - sp->mbcl_active;
9330 m_16kclusters = sp->mbcl_total;
9331 } else if (m_class(i) == MC_MBUF_CL) {
9332 m_mbufclfree = sp->mbcl_total - sp->mbcl_active;
9333 } else if (m_class(i) == MC_MBUF_BIGCL) {
9334 m_mbufbigclfree = sp->mbcl_total - sp->mbcl_active;
9335 } else if (njcl > 0 && m_class(i) == MC_MBUF_16KCL) {
9336 m_mbuf16kclfree = sp->mbcl_total - sp->mbcl_active;
9337 }
9338
9339 mem = sp->mbcl_ctotal * sp->mbcl_size;
9340 totmem += mem;
9341 totfree += (sp->mbcl_mc_cached + sp->mbcl_infree) *
9342 sp->mbcl_size;
9343 totreturned += sp->mbcl_release_cnt;
9344 }
9345
9346 /* adjust free counts to include composite caches */
9347 m_clfree += m_mbufclfree;
9348 m_bigclfree += m_mbufbigclfree;
9349 m_16kclfree += m_mbuf16kclfree;
9350
9351 totmbufs = 0;
9352 for (mp = mbtypes; mp->mt_name != NULL; mp++) {
9353 totmbufs += mbstat.m_mtypes[mp->mt_type];
9354 }
9355 if (totmbufs > m_mbufs) {
9356 totmbufs = m_mbufs;
9357 }
9358 k = scnprintf(c, clen, "%lu/%u mbufs in use:\n", totmbufs, m_mbufs);
9359 MBUF_DUMP_BUF_CHK();
9360
9361 bzero(&seen, sizeof(seen));
9362 for (mp = mbtypes; mp->mt_name != NULL; mp++) {
9363 if (mbstat.m_mtypes[mp->mt_type] != 0) {
9364 seen[mp->mt_type] = 1;
9365 k = scnprintf(c, clen, "\t%u mbufs allocated to %s\n",
9366 mbstat.m_mtypes[mp->mt_type], mp->mt_name);
9367 MBUF_DUMP_BUF_CHK();
9368 }
9369 }
9370 seen[MT_FREE] = 1;
9371 for (i = 0; i < nmbtypes; i++) {
9372 if (!seen[i] && mbstat.m_mtypes[i] != 0) {
9373 k = scnprintf(c, clen, "\t%u mbufs allocated to "
9374 "<mbuf type %d>\n", mbstat.m_mtypes[i], i);
9375 MBUF_DUMP_BUF_CHK();
9376 }
9377 }
9378 if ((m_mbufs - totmbufs) > 0) {
9379 k = scnprintf(c, clen, "\t%lu mbufs allocated to caches\n",
9380 m_mbufs - totmbufs);
9381 MBUF_DUMP_BUF_CHK();
9382 }
9383 k = scnprintf(c, clen, "%u/%u mbuf 2KB clusters in use\n"
9384 "%u/%u mbuf 4KB clusters in use\n",
9385 (unsigned int)(mbstat.m_clusters - m_clfree),
9386 (unsigned int)mbstat.m_clusters,
9387 (unsigned int)(mbstat.m_bigclusters - m_bigclfree),
9388 (unsigned int)mbstat.m_bigclusters);
9389 MBUF_DUMP_BUF_CHK();
9390
9391 if (njcl > 0) {
9392 k = scnprintf(c, clen, "%u/%u mbuf %uKB clusters in use\n",
9393 m_16kclusters - m_16kclfree, m_16kclusters,
9394 njclbytes / 1024);
9395 MBUF_DUMP_BUF_CHK();
9396 }
9397 totused = totmem - totfree;
9398 if (totmem == 0) {
9399 totpct = 0;
9400 } else if (totused < (ULONG_MAX / 100)) {
9401 totpct = (totused * 100) / totmem;
9402 } else {
9403 u_long totmem1 = totmem / 100;
9404 u_long totused1 = totused / 100;
9405 totpct = (totused1 * 100) / totmem1;
9406 }
9407 k = scnprintf(c, clen, "%lu KB allocated to network (approx. %lu%% "
9408 "in use)\n", totmem / 1024, totpct);
9409 MBUF_DUMP_BUF_CHK();
9410 k = scnprintf(c, clen, "%lu KB returned to the system\n",
9411 totreturned / 1024);
9412 MBUF_DUMP_BUF_CHK();
9413
9414 net_update_uptime();
9415
9416 k = scnprintf(c, clen,
9417 "worker thread runs: %u, expansions: %llu, cl %llu/%llu, "
9418 "bigcl %llu/%llu, 16k %llu/%llu\n", mbuf_worker_run_cnt,
9419 mb_expand_cnt, mb_expand_cl_cnt, mb_expand_cl_total,
9420 mb_expand_bigcl_cnt, mb_expand_bigcl_total, mb_expand_16kcl_cnt,
9421 mb_expand_16kcl_total);
9422 MBUF_DUMP_BUF_CHK();
9423 if (mbuf_worker_last_runtime != 0) {
9424 k = scnprintf(c, clen, "worker thread last run time: "
9425 "%llu (%llu seconds ago)\n",
9426 mbuf_worker_last_runtime,
9427 net_uptime() - mbuf_worker_last_runtime);
9428 MBUF_DUMP_BUF_CHK();
9429 }
9430 if (mbuf_drain_last_runtime != 0) {
9431 k = scnprintf(c, clen, "drain routine last run time: "
9432 "%llu (%llu seconds ago)\n",
9433 mbuf_drain_last_runtime,
9434 net_uptime() - mbuf_drain_last_runtime);
9435 MBUF_DUMP_BUF_CHK();
9436 }
9437
9438 /*
9439 * Log where the most mbufs have accumulated:
9440 * - Process socket buffers
9441 * - TCP reassembly queue
9442 * - Interface AQM queue (output) and DLIL input queue
9443 */
9444 args.non_blocking = true;
9445 proc_iterate(PROC_ALLPROCLIST,
9446 mbuf_watchdog_defunct_iterate, &args, NULL, NULL);
9447 if (args.top_app != NULL) {
9448 k = scnprintf(c, clen, "\ntop proc mbuf space %u bytes by %s:%d\n",
9449 args.top_app_space_used,
9450 proc_name_address(args.top_app),
9451 proc_pid(args.top_app));
9452 proc_rele(args.top_app);
9453 }
9454 MBUF_DUMP_BUF_CHK();
9455
9456 #if INET
9457 k = dump_tcp_reass_qlen(c, clen);
9458 MBUF_DUMP_BUF_CHK();
9459 #endif /* INET */
9460
9461 #if MPTCP
9462 k = dump_mptcp_reass_qlen(c, clen);
9463 MBUF_DUMP_BUF_CHK();
9464 #endif /* MPTCP */
9465
9466 #if NETWORKING
9467 k = dlil_dump_top_if_qlen(c, clen);
9468 MBUF_DUMP_BUF_CHK();
9469 #endif /* NETWORKING */
9470
9471 /* mbuf leak detection statistics */
9472 mleak_update_stats();
9473
9474 k = scnprintf(c, clen, "\nmbuf leak detection table:\n");
9475 MBUF_DUMP_BUF_CHK();
9476 k = scnprintf(c, clen, "\ttotal captured: %u (one per %u)\n",
9477 mleak_table.mleak_capture / mleak_table.mleak_sample_factor,
9478 mleak_table.mleak_sample_factor);
9479 MBUF_DUMP_BUF_CHK();
9480 k = scnprintf(c, clen, "\ttotal allocs outstanding: %llu\n",
9481 mleak_table.outstanding_allocs);
9482 MBUF_DUMP_BUF_CHK();
9483 k = scnprintf(c, clen, "\tnew hash recorded: %llu allocs, %llu traces\n",
9484 mleak_table.alloc_recorded, mleak_table.trace_recorded);
9485 MBUF_DUMP_BUF_CHK();
9486 k = scnprintf(c, clen, "\thash collisions: %llu allocs, %llu traces\n",
9487 mleak_table.alloc_collisions, mleak_table.trace_collisions);
9488 MBUF_DUMP_BUF_CHK();
9489 k = scnprintf(c, clen, "\toverwrites: %llu allocs, %llu traces\n",
9490 mleak_table.alloc_overwrites, mleak_table.trace_overwrites);
9491 MBUF_DUMP_BUF_CHK();
9492 k = scnprintf(c, clen, "\tlock conflicts: %llu\n\n",
9493 mleak_table.total_conflicts);
9494 MBUF_DUMP_BUF_CHK();
9495
9496 k = scnprintf(c, clen, "top %d outstanding traces:\n",
9497 mleak_stat->ml_cnt);
9498 MBUF_DUMP_BUF_CHK();
9499 for (i = 0; i < mleak_stat->ml_cnt; i++) {
9500 mltr = &mleak_stat->ml_trace[i];
9501 k = scnprintf(c, clen, "[%d] %llu outstanding alloc(s), "
9502 "%llu hit(s), %llu collision(s)\n", (i + 1),
9503 mltr->mltr_allocs, mltr->mltr_hitcount,
9504 mltr->mltr_collisions);
9505 MBUF_DUMP_BUF_CHK();
9506 }
9507
9508 if (mleak_stat->ml_isaddr64) {
9509 k = scnprintf(c, clen, MB_LEAK_HDR_64);
9510 } else {
9511 k = scnprintf(c, clen, MB_LEAK_HDR_32);
9512 }
9513 MBUF_DUMP_BUF_CHK();
9514
9515 for (i = 0; i < MLEAK_STACK_DEPTH; i++) {
9516 k = scnprintf(c, clen, "%2d: ", (i + 1));
9517 MBUF_DUMP_BUF_CHK();
9518 for (j = 0; j < mleak_stat->ml_cnt; j++) {
9519 mltr = &mleak_stat->ml_trace[j];
9520 if (i < mltr->mltr_depth) {
9521 if (mleak_stat->ml_isaddr64) {
9522 k = scnprintf(c, clen, "0x%0llx ",
9523 (uint64_t)VM_KERNEL_UNSLIDE(
9524 mltr->mltr_addr[i]));
9525 } else {
9526 k = scnprintf(c, clen,
9527 "0x%08x ",
9528 (uint32_t)VM_KERNEL_UNSLIDE(
9529 mltr->mltr_addr[i]));
9530 }
9531 } else {
9532 if (mleak_stat->ml_isaddr64) {
9533 k = scnprintf(c, clen,
9534 MB_LEAK_SPACING_64);
9535 } else {
9536 k = scnprintf(c, clen,
9537 MB_LEAK_SPACING_32);
9538 }
9539 }
9540 MBUF_DUMP_BUF_CHK();
9541 }
9542 k = scnprintf(c, clen, "\n");
9543 MBUF_DUMP_BUF_CHK();
9544 }
9545
9546 done:
9547 return mbuf_dump_buf;
9548 }
9549
9550 #undef MBUF_DUMP_BUF_CHK
9551 #endif /* CONFIG_MBUF_MCACHE */
9552
9553 /*
9554 * Convert between a regular and a packet header mbuf. Caller is responsible
9555 * for setting or clearing M_PKTHDR; this routine does the rest of the work.
9556 */
9557 int
9558 m_reinit(struct mbuf *m, int hdr)
9559 {
9560 int ret = 0;
9561
9562 if (hdr) {
9563 VERIFY(!(m->m_flags & M_PKTHDR));
9564 if (!(m->m_flags & M_EXT) &&
9565 (m->m_data != (uintptr_t)m->m_dat || m->m_len > 0)) {
9566 /*
9567 * If there's no external cluster attached and the
9568 * mbuf appears to contain user data, we cannot
9569 * safely convert this to a packet header mbuf,
9570 * as the packet header structure might overlap
9571 * with the data.
9572 */
9573 printf("%s: cannot set M_PKTHDR on altered mbuf %llx, "
9574 "m_data %llx (expected %llx), "
9575 "m_len %d (expected 0)\n",
9576 __func__,
9577 (uint64_t)VM_KERNEL_ADDRPERM((uintptr_t)m),
9578 (uint64_t)VM_KERNEL_ADDRPERM((uintptr_t)m->m_data),
9579 (uint64_t)VM_KERNEL_ADDRPERM((uintptr_t)(m->m_dat)), m->m_len);
9580 ret = EBUSY;
9581 } else {
9582 VERIFY((m->m_flags & M_EXT) || m->m_data == (uintptr_t)m->m_dat);
9583 m->m_flags |= M_PKTHDR;
9584 MBUF_INIT_PKTHDR(m);
9585 }
9586 } else {
9587 /* Check for scratch area overflow */
9588 m_redzone_verify(m);
9589 /* Free the aux data and tags if there is any */
9590 m_tag_delete_chain(m);
9591 m_do_tx_compl_callback(m, NULL);
9592 m->m_flags &= ~M_PKTHDR;
9593 }
9594
9595 return ret;
9596 }
9597
9598 int
9599 m_ext_set_prop(struct mbuf *m, uint32_t o, uint32_t n)
9600 {
9601 ASSERT(m->m_flags & M_EXT);
9602 return os_atomic_cmpxchg(&MEXT_PRIV(m), o, n, acq_rel);
9603 }
9604
9605 uint32_t
9606 m_ext_get_prop(struct mbuf *m)
9607 {
9608 ASSERT(m->m_flags & M_EXT);
9609 return MEXT_PRIV(m);
9610 }
9611
9612 int
9613 m_ext_paired_is_active(struct mbuf *m)
9614 {
9615 return MBUF_IS_PAIRED(m) ? (MEXT_PREF(m) > MEXT_MINREF(m)) : 1;
9616 }
9617
9618 void
9619 m_ext_paired_activate(struct mbuf *m)
9620 {
9621 struct ext_ref *rfa;
9622 int hdr, type;
9623 caddr_t extbuf;
9624 m_ext_free_func_t extfree;
9625 u_int extsize;
9626
9627 VERIFY(MBUF_IS_PAIRED(m));
9628 VERIFY(MEXT_REF(m) == MEXT_MINREF(m));
9629 VERIFY(MEXT_PREF(m) == MEXT_MINREF(m));
9630
9631 hdr = (m->m_flags & M_PKTHDR);
9632 type = m->m_type;
9633 extbuf = m->m_ext.ext_buf;
9634 extfree = m_get_ext_free(m);
9635 extsize = m->m_ext.ext_size;
9636 rfa = m_get_rfa(m);
9637
9638 VERIFY(extbuf != NULL && rfa != NULL);
9639
9640 /*
9641 * Safe to reinitialize packet header tags, since it's
9642 * already taken care of at m_free() time. Similar to
9643 * what's done in m_clattach() for the cluster. Bump
9644 * up MEXT_PREF to indicate activation.
9645 */
9646 MBUF_INIT(m, hdr, type);
9647 MEXT_INIT(m, extbuf, extsize, extfree, (caddr_t)m, rfa,
9648 1, 1, 2, EXTF_PAIRED, MEXT_PRIV(m), m);
9649 }
9650
9651 void
9652 m_scratch_init(struct mbuf *m)
9653 {
9654 struct pkthdr *pkt = &m->m_pkthdr;
9655
9656 VERIFY(m->m_flags & M_PKTHDR);
9657
9658 /* See comments in <rdar://problem/14040693> */
9659 if (pkt->pkt_flags & PKTF_PRIV_GUARDED) {
9660 panic_plain("Invalid attempt to modify guarded module-private "
9661 "area: mbuf %p, pkt_flags 0x%x\n", m, pkt->pkt_flags);
9662 /* NOTREACHED */
9663 }
9664
9665 bzero(&pkt->pkt_mpriv, sizeof(pkt->pkt_mpriv));
9666 }
9667
9668 /*
9669 * This routine is reserved for mbuf_get_driver_scratch(); clients inside
9670 * xnu that intend on utilizing the module-private area should directly
9671 * refer to the pkt_mpriv structure in the pkthdr. They are also expected
9672 * to set and clear PKTF_PRIV_GUARDED, while owning the packet and prior
9673 * to handing it off to another module, respectively.
9674 */
9675 u_int32_t
9676 m_scratch_get(struct mbuf *m, u_int8_t **p)
9677 {
9678 struct pkthdr *pkt = &m->m_pkthdr;
9679
9680 VERIFY(m->m_flags & M_PKTHDR);
9681
9682 /* See comments in <rdar://problem/14040693> */
9683 if (pkt->pkt_flags & PKTF_PRIV_GUARDED) {
9684 panic_plain("Invalid attempt to access guarded module-private "
9685 "area: mbuf %p, pkt_flags 0x%x\n", m, pkt->pkt_flags);
9686 /* NOTREACHED */
9687 }
9688
9689 #if CONFIG_MBUF_MCACHE
9690 if (mcltrace) {
9691 mcache_audit_t *mca;
9692
9693 lck_mtx_lock(mbuf_mlock);
9694 mca = mcl_audit_buf2mca(MC_MBUF, (mcache_obj_t *)m);
9695 if (mca->mca_uflags & MB_SCVALID) {
9696 mcl_audit_scratch(mca);
9697 }
9698 lck_mtx_unlock(mbuf_mlock);
9699 }
9700 #endif /* CONFIG_MBUF_MCACHE */
9701
9702 *p = (u_int8_t *)&pkt->pkt_mpriv;
9703 return sizeof(pkt->pkt_mpriv);
9704 }
9705
9706 void
9707 m_add_crumb(struct mbuf *m, uint16_t crumb)
9708 {
9709 VERIFY(m->m_flags & M_PKTHDR);
9710
9711 m->m_pkthdr.pkt_crumbs |= crumb;
9712 }
9713
9714 static void
9715 m_redzone_init(struct mbuf *m)
9716 {
9717 VERIFY(m->m_flags & M_PKTHDR);
9718 /*
9719 * Each mbuf has a unique red zone pattern, which is a XOR
9720 * of the red zone cookie and the address of the mbuf.
9721 */
9722 m->m_pkthdr.redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
9723 }
9724
9725 static void
9726 m_redzone_verify(struct mbuf *m)
9727 {
9728 u_int32_t mb_redzone;
9729
9730 VERIFY(m->m_flags & M_PKTHDR);
9731
9732 mb_redzone = ((u_int32_t)(uintptr_t)m) ^ mb_redzone_cookie;
9733 if (m->m_pkthdr.redzone != mb_redzone) {
9734 panic("mbuf %p redzone violation with value 0x%x "
9735 "(instead of 0x%x, using cookie 0x%x)\n",
9736 m, m->m_pkthdr.redzone, mb_redzone, mb_redzone_cookie);
9737 /* NOTREACHED */
9738 }
9739 }
9740
9741 __private_extern__ inline void
9742 m_set_ext(struct mbuf *m, struct ext_ref *rfa, m_ext_free_func_t ext_free,
9743 caddr_t ext_arg)
9744 {
9745 VERIFY(m->m_flags & M_EXT);
9746 if (rfa != NULL) {
9747 m_set_rfa(m, rfa);
9748 if (ext_free != NULL) {
9749 rfa->ext_token = ((uintptr_t)&rfa->ext_token) ^
9750 mb_obscure_extfree;
9751 uintptr_t ext_free_val = ptrauth_nop_cast(uintptr_t, ext_free) ^ rfa->ext_token;
9752 m->m_ext.ext_free = ptrauth_nop_cast(m_ext_free_func_t, ext_free_val);
9753 if (ext_arg != NULL) {
9754 m->m_ext.ext_arg =
9755 (caddr_t)(((uintptr_t)ext_arg) ^ rfa->ext_token);
9756 } else {
9757 m->m_ext.ext_arg = NULL;
9758 }
9759 } else {
9760 rfa->ext_token = 0;
9761 m->m_ext.ext_free = NULL;
9762 m->m_ext.ext_arg = NULL;
9763 }
9764 } else {
9765 /*
9766 * If we are going to loose the cookie in ext_token by
9767 * resetting the rfa, we should use the global cookie
9768 * to obscure the ext_free and ext_arg pointers.
9769 */
9770 if (ext_free != NULL) {
9771 uintptr_t ext_free_val = ptrauth_nop_cast(uintptr_t, ext_free) ^ mb_obscure_extfree;
9772 m->m_ext.ext_free = ptrauth_nop_cast(m_ext_free_func_t, ext_free_val);
9773 if (ext_arg != NULL) {
9774 m->m_ext.ext_arg =
9775 (caddr_t)((uintptr_t)ext_arg ^
9776 mb_obscure_extfree);
9777 } else {
9778 m->m_ext.ext_arg = NULL;
9779 }
9780 } else {
9781 m->m_ext.ext_free = NULL;
9782 m->m_ext.ext_arg = NULL;
9783 }
9784 m->m_ext.ext_refflags = NULL;
9785 }
9786 }
9787
9788 __private_extern__ inline struct ext_ref *
9789 m_get_rfa(struct mbuf *m)
9790 {
9791 if (m->m_ext.ext_refflags == NULL) {
9792 return NULL;
9793 } else {
9794 return (struct ext_ref *)(((uintptr_t)m->m_ext.ext_refflags) ^ mb_obscure_extref);
9795 }
9796 }
9797
9798 static inline void
9799 m_set_rfa(struct mbuf *m, struct ext_ref *rfa)
9800 {
9801 if (rfa != NULL) {
9802 m->m_ext.ext_refflags =
9803 (struct ext_ref *)(((uintptr_t)rfa) ^ mb_obscure_extref);
9804 } else {
9805 m->m_ext.ext_refflags = NULL;
9806 }
9807 }
9808
9809 __private_extern__ inline m_ext_free_func_t
9810 m_get_ext_free(struct mbuf *m)
9811 {
9812 struct ext_ref *rfa;
9813 if (m->m_ext.ext_free == NULL) {
9814 return NULL;
9815 }
9816
9817 rfa = m_get_rfa(m);
9818 if (rfa == NULL) {
9819 uintptr_t ext_free_val = ptrauth_nop_cast(uintptr_t, m->m_ext.ext_free) ^ mb_obscure_extfree;
9820 return ptrauth_nop_cast(m_ext_free_func_t, ext_free_val);
9821 } else {
9822 uintptr_t ext_free_val = ptrauth_nop_cast(uintptr_t, m->m_ext.ext_free) ^ rfa->ext_token;
9823 return ptrauth_nop_cast(m_ext_free_func_t, ext_free_val);
9824 }
9825 }
9826
9827 __private_extern__ inline caddr_t
9828 m_get_ext_arg(struct mbuf *m)
9829 {
9830 struct ext_ref *rfa;
9831 if (m->m_ext.ext_arg == NULL) {
9832 return NULL;
9833 }
9834
9835 rfa = m_get_rfa(m);
9836 if (rfa == NULL) {
9837 return (caddr_t)((uintptr_t)m->m_ext.ext_arg ^ mb_obscure_extfree);
9838 } else {
9839 return (caddr_t)(((uintptr_t)m->m_ext.ext_arg) ^
9840 rfa->ext_token);
9841 }
9842 }
9843
9844 #if CONFIG_MBUF_MCACHE
9845 /*
9846 * Simple routine to avoid taking the lock when we can't run the
9847 * mbuf drain.
9848 */
9849 static int
9850 mbuf_drain_checks(boolean_t ignore_waiters)
9851 {
9852 if (mb_drain_maxint == 0) {
9853 return 0;
9854 }
9855 if (!ignore_waiters && mb_waiters != 0) {
9856 return 0;
9857 }
9858
9859 return 1;
9860 }
9861
9862 /*
9863 * Called by the VM when there's memory pressure or when we exhausted
9864 * the 4k/16k reserved space.
9865 */
9866 static void
9867 mbuf_drain_locked(boolean_t ignore_waiters)
9868 {
9869 mbuf_class_t mc;
9870 mcl_slab_t *sp, *sp_tmp, *nsp;
9871 unsigned int num, k, interval, released = 0;
9872 unsigned long total_mem = 0, use_mem = 0;
9873 boolean_t ret, purge_caches = FALSE;
9874 ppnum_t offset;
9875 mcache_obj_t *obj;
9876 unsigned long per;
9877 static unsigned char scratch[32];
9878 static ppnum_t scratch_pa = 0;
9879
9880 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
9881 if (!mbuf_drain_checks(ignore_waiters)) {
9882 return;
9883 }
9884 if (scratch_pa == 0) {
9885 bzero(scratch, sizeof(scratch));
9886 scratch_pa = pmap_find_phys(kernel_pmap, (addr64_t)scratch);
9887 VERIFY(scratch_pa);
9888 } else if (mclverify) {
9889 /*
9890 * Panic if a driver wrote to our scratch memory.
9891 */
9892 for (k = 0; k < sizeof(scratch); k++) {
9893 if (scratch[k]) {
9894 panic("suspect DMA to freed address");
9895 }
9896 }
9897 }
9898 /*
9899 * Don't free memory too often as that could cause excessive
9900 * waiting times for mbufs. Purge caches if we were asked to drain
9901 * in the last 5 minutes.
9902 */
9903 if (mbuf_drain_last_runtime != 0) {
9904 interval = net_uptime() - mbuf_drain_last_runtime;
9905 if (interval <= mb_drain_maxint) {
9906 return;
9907 }
9908 if (interval <= mb_drain_maxint * 5) {
9909 purge_caches = TRUE;
9910 }
9911 }
9912 mbuf_drain_last_runtime = net_uptime();
9913 /*
9914 * Don't free any memory if we're using 60% or more.
9915 */
9916 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
9917 total_mem += m_total(mc) * m_maxsize(mc);
9918 use_mem += m_active(mc) * m_maxsize(mc);
9919 }
9920 per = (use_mem * 100) / total_mem;
9921 if (per >= 60) {
9922 return;
9923 }
9924 /*
9925 * Purge all the caches. This effectively disables
9926 * caching for a few seconds, but the mbuf worker thread will
9927 * re-enable them again.
9928 */
9929 if (purge_caches == TRUE) {
9930 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
9931 if (m_total(mc) < m_avgtotal(mc)) {
9932 continue;
9933 }
9934 lck_mtx_unlock(mbuf_mlock);
9935 ret = mcache_purge_cache(m_cache(mc), FALSE);
9936 lck_mtx_lock(mbuf_mlock);
9937 if (ret == TRUE) {
9938 m_purge_cnt(mc)++;
9939 }
9940 }
9941 }
9942 /*
9943 * Move the objects from the composite class freelist to
9944 * the rudimentary slabs list, but keep at least 10% of the average
9945 * total in the freelist.
9946 */
9947 for (mc = 0; mc < NELEM(mbuf_table); mc++) {
9948 while (m_cobjlist(mc) &&
9949 m_total(mc) < m_avgtotal(mc) &&
9950 m_infree(mc) > 0.1 * m_avgtotal(mc) + m_minlimit(mc)) {
9951 obj = m_cobjlist(mc);
9952 m_cobjlist(mc) = obj->obj_next;
9953 obj->obj_next = NULL;
9954 num = cslab_free(mc, obj, 1);
9955 VERIFY(num == 1);
9956 m_free_cnt(mc)++;
9957 m_infree(mc)--;
9958 /* cslab_free() handles m_total */
9959 }
9960 }
9961 /*
9962 * Free the buffers present in the slab list up to 10% of the total
9963 * average per class.
9964 *
9965 * We walk the list backwards in an attempt to reduce fragmentation.
9966 */
9967 for (mc = NELEM(mbuf_table) - 1; (int)mc >= 0; mc--) {
9968 TAILQ_FOREACH_SAFE(sp, &m_slablist(mc), sl_link, sp_tmp) {
9969 /*
9970 * Process only unused slabs occupying memory.
9971 */
9972 if (sp->sl_refcnt != 0 || sp->sl_len == 0 ||
9973 sp->sl_base == NULL) {
9974 continue;
9975 }
9976 if (m_total(mc) < m_avgtotal(mc) ||
9977 m_infree(mc) < 0.1 * m_avgtotal(mc) + m_minlimit(mc)) {
9978 break;
9979 }
9980 slab_remove(sp, mc);
9981 switch (mc) {
9982 case MC_MBUF:
9983 m_infree(mc) -= NMBPG;
9984 m_total(mc) -= NMBPG;
9985 if (mclaudit != NULL) {
9986 mcl_audit_free(sp->sl_base, NMBPG);
9987 }
9988 break;
9989 case MC_CL:
9990 m_infree(mc) -= NCLPG;
9991 m_total(mc) -= NCLPG;
9992 if (mclaudit != NULL) {
9993 mcl_audit_free(sp->sl_base, NMBPG);
9994 }
9995 break;
9996 case MC_BIGCL:
9997 {
9998 m_infree(mc) -= NBCLPG;
9999 m_total(mc) -= NBCLPG;
10000 if (mclaudit != NULL) {
10001 mcl_audit_free(sp->sl_base, NMBPG);
10002 }
10003 break;
10004 }
10005 case MC_16KCL:
10006 m_infree(mc)--;
10007 m_total(mc)--;
10008 for (nsp = sp, k = 1; k < NSLABSP16KB; k++) {
10009 nsp = nsp->sl_next;
10010 VERIFY(nsp->sl_refcnt == 0 &&
10011 nsp->sl_base != NULL &&
10012 nsp->sl_len == 0);
10013 slab_init(nsp, 0, 0, NULL, NULL, 0, 0,
10014 0);
10015 nsp->sl_flags = 0;
10016 }
10017 if (mclaudit != NULL) {
10018 if (sp->sl_len == PAGE_SIZE) {
10019 mcl_audit_free(sp->sl_base,
10020 NMBPG);
10021 } else {
10022 mcl_audit_free(sp->sl_base, 1);
10023 }
10024 }
10025 break;
10026 default:
10027 /*
10028 * The composite classes have their own
10029 * freelist (m_cobjlist), so we only
10030 * process rudimentary classes here.
10031 */
10032 VERIFY(0);
10033 }
10034 m_release_cnt(mc) += m_size(mc);
10035 released += m_size(mc);
10036 VERIFY(sp->sl_base != NULL &&
10037 sp->sl_len >= PAGE_SIZE);
10038 offset = MTOPG(sp->sl_base);
10039 /*
10040 * Make sure the IOMapper points to a valid, but
10041 * bogus, address. This should prevent further DMA
10042 * accesses to freed memory.
10043 */
10044 IOMapperInsertPage(mcl_paddr_base, offset, scratch_pa);
10045 mcl_paddr[offset] = 0;
10046 kmem_free(mb_map, (vm_offset_t)sp->sl_base,
10047 sp->sl_len);
10048 slab_init(sp, 0, 0, NULL, NULL, 0, 0, 0);
10049 sp->sl_flags = 0;
10050 }
10051 }
10052 mbstat.m_drain++;
10053 mbstat.m_bigclusters = m_total(MC_BIGCL);
10054 mbstat.m_clusters = m_total(MC_CL);
10055 mbstat.m_mbufs = m_total(MC_MBUF);
10056 mbuf_stat_sync();
10057 mbuf_mtypes_sync(TRUE);
10058 }
10059
10060 __private_extern__ void
10061 mbuf_drain(boolean_t ignore_waiters)
10062 {
10063 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_NOTOWNED);
10064 if (!mbuf_drain_checks(ignore_waiters)) {
10065 return;
10066 }
10067 lck_mtx_lock(mbuf_mlock);
10068 mbuf_drain_locked(ignore_waiters);
10069 lck_mtx_unlock(mbuf_mlock);
10070 }
10071
10072
10073 static int
10074 m_drain_force_sysctl SYSCTL_HANDLER_ARGS
10075 {
10076 #pragma unused(arg1, arg2)
10077 int val = 0, err;
10078
10079 err = sysctl_handle_int(oidp, &val, 0, req);
10080 if (err != 0 || req->newptr == USER_ADDR_NULL) {
10081 return err;
10082 }
10083 if (val) {
10084 mbuf_drain(TRUE);
10085 }
10086
10087 return err;
10088 }
10089
10090 #if DEBUG || DEVELOPMENT
10091 __printflike(3, 4)
10092 static void
10093 _mbwdog_logger(const char *func, const int line, const char *fmt, ...)
10094 {
10095 va_list ap;
10096 struct timeval now;
10097 char str[384], p[256];
10098 int len;
10099
10100 LCK_MTX_ASSERT(mbuf_mlock, LCK_MTX_ASSERT_OWNED);
10101 if (mbwdog_logging == NULL) {
10102 /*
10103 * This might block under a mutex, which isn't really great,
10104 * but this happens once, so we'll live.
10105 */
10106 mbwdog_logging = zalloc_permanent(mbwdog_logging_size,
10107 ZALIGN_NONE);
10108 }
10109 va_start(ap, fmt);
10110 vsnprintf(p, sizeof(p), fmt, ap);
10111 va_end(ap);
10112 microuptime(&now);
10113 len = scnprintf(str, sizeof(str),
10114 "\n%ld.%d (%d/%llx) %s:%d %s",
10115 now.tv_sec, now.tv_usec,
10116 proc_getpid(current_proc()),
10117 (uint64_t)VM_KERNEL_ADDRPERM(current_thread()),
10118 func, line, p);
10119 if (len < 0) {
10120 return;
10121 }
10122 if (mbwdog_logging_used + len > mbwdog_logging_size) {
10123 mbwdog_logging_used = mbwdog_logging_used / 2;
10124 memmove(mbwdog_logging, mbwdog_logging + mbwdog_logging_used,
10125 mbwdog_logging_size - mbwdog_logging_used);
10126 mbwdog_logging[mbwdog_logging_used] = 0;
10127 }
10128 strlcat(mbwdog_logging, str, mbwdog_logging_size);
10129 mbwdog_logging_used += len;
10130 }
10131
10132 #endif // DEBUG || DEVELOPMENT
10133
10134 static void
10135 mtracelarge_register(size_t size)
10136 {
10137 int i;
10138 struct mtracelarge *trace;
10139 uintptr_t bt[MLEAK_STACK_DEPTH];
10140 unsigned int depth;
10141
10142 depth = backtrace(bt, MLEAK_STACK_DEPTH, NULL, NULL);
10143 /* Check if this entry is already on the list. */
10144 for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
10145 trace = &mtracelarge_table[i];
10146 if (trace->size == size && trace->depth == depth &&
10147 memcmp(bt, trace->addr, depth * sizeof(uintptr_t)) == 0) {
10148 return;
10149 }
10150 }
10151 for (i = 0; i < MTRACELARGE_NUM_TRACES; i++) {
10152 trace = &mtracelarge_table[i];
10153 if (size > trace->size) {
10154 trace->depth = depth;
10155 memcpy(trace->addr, bt, depth * sizeof(uintptr_t));
10156 trace->size = size;
10157 break;
10158 }
10159 }
10160 }
10161
10162 #if DEBUG || DEVELOPMENT
10163
10164 static int
10165 mbuf_wd_dump_sysctl SYSCTL_HANDLER_ARGS
10166 {
10167 char *str;
10168
10169 ifnet_head_lock_shared();
10170 lck_mtx_lock(mbuf_mlock);
10171
10172 str = mbuf_dump();
10173
10174 lck_mtx_unlock(mbuf_mlock);
10175 ifnet_head_done();
10176
10177 return sysctl_io_string(req, str, 0, 0, NULL);
10178 }
10179
10180 #endif /* DEBUG || DEVELOPMENT */
10181 #endif /* CONFIG_MBUF_MCACHE */
10182
10183 SYSCTL_DECL(_kern_ipc);
10184 #if DEBUG || DEVELOPMENT
10185 #if SKYWALK && CONFIG_MBUF_MCACHE
10186 SYSCTL_UINT(_kern_ipc, OID_AUTO, mc_threshold_scale_factor,
10187 CTLFLAG_RW | CTLFLAG_LOCKED, &mc_threshold_scale_down_factor,
10188 MC_THRESHOLD_SCALE_DOWN_FACTOR,
10189 "scale down factor for mbuf cache thresholds");
10190 #endif /* SKYWALK && CONFIG_MBUF_MCACHE */
10191 #if CONFIG_MBUF_MCACHE
10192 SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_wd_dump,
10193 CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_LOCKED,
10194 0, 0, mbuf_wd_dump_sysctl, "A", "mbuf watchdog dump");
10195 #endif /* CONFIG_MBUF_MCACHE */
10196 #endif /* DEBUG || DEVELOPMENT */
10197 SYSCTL_PROC(_kern_ipc, KIPC_MBSTAT, mbstat,
10198 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
10199 0, 0, mbstat_sysctl, "S,mbstat", "");
10200 SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_stat,
10201 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
10202 0, 0, mb_stat_sysctl, "S,mb_stat", "");
10203 #if CONFIG_MBUF_MCACHE
10204 SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_top_trace,
10205 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
10206 0, 0, mleak_top_trace_sysctl, "S,mb_top_trace", "");
10207 SYSCTL_PROC(_kern_ipc, OID_AUTO, mleak_table,
10208 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
10209 0, 0, mleak_table_sysctl, "S,mleak_table", "");
10210 SYSCTL_INT(_kern_ipc, OID_AUTO, mleak_sample_factor,
10211 CTLFLAG_RW | CTLFLAG_LOCKED, &mleak_table.mleak_sample_factor, 0, "");
10212 SYSCTL_INT(_kern_ipc, OID_AUTO, mb_normalized,
10213 CTLFLAG_RD | CTLFLAG_LOCKED, &mb_normalized, 0, "");
10214 SYSCTL_INT(_kern_ipc, OID_AUTO, mb_watchdog,
10215 CTLFLAG_RW | CTLFLAG_LOCKED, &mb_watchdog, 0, "");
10216 SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_drain_force,
10217 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
10218 m_drain_force_sysctl, "I",
10219 "Forces the mbuf garbage collection to run");
10220 SYSCTL_INT(_kern_ipc, OID_AUTO, mb_drain_maxint,
10221 CTLFLAG_RW | CTLFLAG_LOCKED, &mb_drain_maxint, 0,
10222 "Minimum time interval between garbage collection");
10223 #endif /* CONFIG_MBUF_MCACHE */
10224 SYSCTL_INT(_kern_ipc, OID_AUTO, mb_memory_pressure_percentage,
10225 CTLFLAG_RW | CTLFLAG_LOCKED, &mb_memory_pressure_percentage, 0,
10226 "Percentage of when we trigger memory-pressure for an mbuf-class");
10227 #if CONFIG_MBUF_MCACHE
10228 static int mb_uses_mcache = 1;
10229 #else
10230 static int mb_uses_mcache = 0;
10231 #endif /* CONFIG_MBUF_MCACHE */
10232 SYSCTL_INT(_kern_ipc, OID_AUTO, mb_uses_mcache,
10233 CTLFLAG_LOCKED, &mb_uses_mcache, 0,
10234 "Whether mbufs use mcache");
10235