1 /*
2 * Copyright (c) 2000-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */
29
30 /*
31 * Copyright (C) 1999 WIDE Project.
32 * All rights reserved.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. Neither the name of the project nor the names of its contributors
43 * may be used to endorse or promote products derived from this software
44 * without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 */
58
59 /*
60 * Copyright (c) 1982, 1986, 1988, 1991, 1993
61 * The Regents of the University of California. All rights reserved.
62 *
63 * Redistribution and use in source and binary forms, with or without
64 * modification, are permitted provided that the following conditions
65 * are met:
66 * 1. Redistributions of source code must retain the above copyright
67 * notice, this list of conditions and the following disclaimer.
68 * 2. Redistributions in binary form must reproduce the above copyright
69 * notice, this list of conditions and the following disclaimer in the
70 * documentation and/or other materials provided with the distribution.
71 * 3. All advertising materials mentioning features or use of this software
72 * must display the following acknowledgement:
73 * This product includes software developed by the University of
74 * California, Berkeley and its contributors.
75 * 4. Neither the name of the University nor the names of its contributors
76 * may be used to endorse or promote products derived from this software
77 * without specific prior written permission.
78 *
79 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
80 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
81 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
82 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
83 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
84 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
85 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
86 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
87 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
88 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
89 * SUCH DAMAGE.
90 *
91 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
92 */
93 /*
94 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
95 * support for mandatory and extensible security protections. This notice
96 * is included in support of clause 2.2 (b) of the Apple Public License,
97 * Version 2.0.
98 */
99
100 /*#define PULLDOWN_DEBUG*/
101
102 #include <sys/param.h>
103 #include <sys/systm.h>
104 #include <sys/proc_internal.h>
105 #include <sys/malloc.h>
106 #include <sys/mbuf.h>
107 #include <sys/mcache.h>
108 #include <sys/sysctl.h>
109
110 #include <netinet/in.h>
111 #include <netinet/ip_var.h>
112 #include <netinet/ip6.h>
113 #include <netinet6/ip6_var.h>
114
115 #include <kern/assert.h>
116
117 #include <os/log.h>
118
119 #include <libkern/OSDebug.h>
120
121 #include <ptrauth.h>
122
123 #if defined(__i386__) || defined(__x86_64__)
124 #define MB_TAG_MBUF 1
125 #endif /* defined(__i386__) || defined(__x86_64__) */
126
127 SYSCTL_DECL(_kern_ipc);
128
129 struct m_tag_type_entry {
130 m_tag_kalloc_func_t mt_alloc_func;
131 m_tag_kfree_func_t mt_free_func;
132 uint16_t mt_type;
133 uint16_t mt_len;
134 };
135
136 typedef struct m_tag_type_entry * __single m_tag_type_entry_ref_t;
137
138 struct m_tag_type_stats {
139 uint64_t mt_alloc_count;
140 uint64_t mt_alloc_failed;
141 uint64_t mt_free_count;
142 };
143
144 typedef struct m_tag_type_stats * __single m_tag_type_stats_ref_t;
145
146 SECURITY_READ_ONLY_LATE(static struct m_tag_type_entry) m_tag_type_table[KERNEL_TAG_TYPE_COUNT] = {};
147
148 static struct m_tag_type_stats m_tag_type_stats[KERNEL_TAG_TYPE_COUNT] = {};
149
150 #ifdef MB_TAG_MBUF
151 static struct m_tag *m_tag_create_mbuf(uint32_t, uint16_t, uint16_t, int, struct mbuf *);
152 #endif /* MB_TAG_MBUF */
153
154 /*
155 * ensure that [off, off + len) is contiguous on the mbuf chain "m".
156 * packet chain before "off" is kept untouched.
157 * if offp == NULL, the target will start at <retval, 0> on resulting chain.
158 * if offp != NULL, the target will start at <retval, *offp> on resulting chain.
159 *
160 * on error return (NULL return value), original "m" will be freed.
161 *
162 * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster)
163 */
164 struct mbuf *
m_pulldown(struct mbuf * m,int off,int len,int * offp)165 m_pulldown(struct mbuf *m, int off, int len, int *offp)
166 {
167 struct mbuf *n = NULL, *o = NULL;
168 int hlen = 0, tlen = 0, olen = 0;
169 int sharedcluster = 0;
170
171 /* check invalid arguments. */
172 VERIFY(len >= 0 && off >= 0);
173
174 if (m == NULL) {
175 panic("m == NULL in m_pulldown()");
176 }
177 if (len > MCLBYTES) {
178 m_freem(m);
179 return NULL; /* impossible */
180 }
181 int tmp_len = 0;
182 if (os_add_overflow(off, len, &tmp_len)) {
183 m_free(m);
184 return NULL;
185 }
186
187 #ifdef PULLDOWN_DEBUG
188 {
189 struct mbuf *t;
190 printf("before:");
191 for (t = m; t; t = t->m_next) {
192 printf(" %d", t->m_len);
193 }
194 printf("\n");
195 }
196 #endif
197 n = m;
198
199 /*
200 * Iterate and make n point to the mbuf
201 * within which the first byte at length
202 * offset is contained from the start of
203 * mbuf chain.
204 */
205 while (n != NULL && off > 0) {
206 if (n->m_len > off) {
207 break;
208 }
209 off -= n->m_len;
210 n = n->m_next;
211 }
212
213 /* be sure to point non-empty mbuf */
214 while (n != NULL && n->m_len == 0) {
215 n = n->m_next;
216 }
217
218 if (!n) {
219 m_freem(m);
220 return NULL; /* mbuf chain too short */
221 }
222
223 /*
224 * the target data is on <n, off>.
225 * if we got enough data on the mbuf "n", we're done.
226 *
227 * It should be noted, that we should only do this either
228 * when offset is 0, i.e. data is pointing to the start
229 * or when the caller specifies an out argument to get
230 * the offset value in the mbuf to work with data pointer
231 * correctly.
232 *
233 * If offset is not 0 and caller did not provide out-argument
234 * to get offset, we should split the mbuf even when the length
235 * is contained in current mbuf.
236 */
237 if ((off == 0 || offp) && len <= n->m_len - off) {
238 goto ok;
239 }
240
241 /*
242 * when len <= n->m_len - off and off != 0, it is a special case.
243 * len bytes from <n, off> sits in single mbuf, but the caller does
244 * not like the starting position (off).
245 * chop the current mbuf into two pieces, set off to 0.
246 */
247 if (len <= n->m_len - off) {
248 o = m_copym(n, off, n->m_len - off, M_DONTWAIT);
249 if (o == NULL) {
250 m_freem(m);
251 return NULL; /* ENOBUFS */
252 }
253 n->m_len = off;
254 o->m_next = n->m_next;
255 n->m_next = o;
256 n = n->m_next;
257 off = 0;
258 goto ok;
259 }
260
261 /*
262 * we need to take hlen from <n, off> and tlen from <n->m_next, 0>,
263 * and construct contiguous mbuf with m_len == len.
264 * note that hlen + tlen == len, and tlen > 0.
265 *
266 * Read these variables as head length and tail length
267 */
268 hlen = n->m_len - off;
269 tlen = len - hlen;
270
271 /*
272 * ensure that we have enough trailing data on mbuf chain.
273 * if not, we can do nothing about the chain.
274 */
275 olen = 0;
276 for (o = n->m_next; o != NULL; o = o->m_next) {
277 olen += o->m_len;
278 }
279 if (hlen + olen < len) {
280 m_freem(m);
281 return NULL; /* mbuf chain too short */
282 }
283
284 /*
285 * easy cases first.
286 * we need to use m_copydata() to get data from <n->m_next, 0>.
287 */
288 if ((n->m_flags & M_EXT) == 0) {
289 sharedcluster = 0;
290 } else {
291 if (m_get_ext_free(n) != NULL) {
292 sharedcluster = 1;
293 } else if (m_mclhasreference(n)) {
294 sharedcluster = 1;
295 } else {
296 sharedcluster = 0;
297 }
298 }
299
300 /*
301 * If we have enough space left in current mbuf to accomodate
302 * tail length, copy tail length worth of data starting with next mbuf
303 * and adjust the length of next one accordingly.
304 */
305 if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen
306 && !sharedcluster) {
307 m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len);
308 n->m_len += tlen;
309 m_adj(n->m_next, tlen);
310 goto ok;
311 }
312
313 /*
314 * If have enough leading space in next mbuf to accomodate head length
315 * of current mbuf, and total resulting length of next mbuf is greater
316 * than or equal to requested len bytes, then just copy hlen from
317 * current to the next one and adjust sizes accordingly.
318 */
319 if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen &&
320 (n->m_next->m_len + hlen) >= len && !sharedcluster) {
321 n->m_next->m_data -= hlen;
322 n->m_next->m_len += hlen;
323 bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), hlen);
324 n->m_len -= hlen;
325 n = n->m_next;
326 off = 0;
327 goto ok;
328 }
329
330 /*
331 * now, we need to do the hard way. don't m_copy as there's no room
332 * on both end.
333 */
334 MGET(o, M_DONTWAIT, m->m_type);
335 if (o == NULL) {
336 m_freem(m);
337 return NULL; /* ENOBUFS */
338 }
339 if (len > MHLEN) { /* use MHLEN just for safety */
340 MCLGET(o, M_DONTWAIT);
341 if ((o->m_flags & M_EXT) == 0) {
342 m_freem(m);
343 m_free(o);
344 return NULL; /* ENOBUFS */
345 }
346 }
347 /* get hlen from <n, off> into <o, 0> */
348 o->m_len = hlen;
349 bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), hlen);
350 n->m_len -= hlen;
351 /* get tlen from <n->m_next, 0> into <o, hlen> */
352 m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len);
353 o->m_len += tlen;
354 m_adj(n->m_next, tlen);
355 o->m_next = n->m_next;
356 n->m_next = o;
357 n = o;
358 off = 0;
359
360 ok:
361 #ifdef PULLDOWN_DEBUG
362 {
363 struct mbuf *t;
364 printf("after:");
365 for (t = m; t; t = t->m_next) {
366 printf("%c%d", t == n ? '*' : ' ', t->m_len);
367 }
368 printf(" (off=%d)\n", off);
369 }
370 #endif
371 if (offp) {
372 *offp = off;
373 }
374 return n;
375 }
376
377 static struct m_tag *
m_tag_kalloc_notsupp(__unused uint32_t id,__unused uint16_t type,__unused uint16_t len,__unused int wait)378 m_tag_kalloc_notsupp(__unused uint32_t id, __unused uint16_t type, __unused uint16_t len, __unused int wait)
379 {
380 return NULL;
381 }
382
383 static void
m_tag_kfree_notsupp(__unused struct m_tag * tag)384 m_tag_kfree_notsupp(__unused struct m_tag *tag)
385 {
386 return;
387 }
388
389 #if defined(HAS_APPLE_PAC)
390 /*
391 * combine into a uintptr_t the m_tag_type that is 16 bits with the m_tag_id is 32 bits
392 */
393 static uintptr_t
m_tag_cookie_from_id_and_type(struct m_tag * tag)394 m_tag_cookie_from_id_and_type(struct m_tag *tag)
395 {
396 uintptr_t cookie;
397
398 #ifdef __LP64__
399 /*
400 * upper 4 bytes: 2 bytes of type
401 * lower 4 bytes: 4 bytes of id
402 */
403 cookie = (((uintptr_t)tag->m_tag_type) << 32) | (uintptr_t)tag->m_tag_id;
404 #else
405 /*
406 * upper 2 bytes: 2 bytes of type or-ed with upper 2 bytes of id
407 * lower 2 bytes: lower 2 bytes of id
408 */
409 cookie = (((uintptr_t)tag->m_tag_type) << 16) | (uintptr_t)tag->m_tag_id;
410 #endif
411 return cookie;
412 }
413
414 void
m_tag_create_cookie(struct m_tag * tag)415 m_tag_create_cookie(struct m_tag *tag)
416 {
417 uintptr_t cookie = m_tag_cookie_from_id_and_type(tag);
418
419 tag->m_tag_cookie = (uintptr_t) ptrauth_sign_unauthenticated((void *)cookie,
420 ptrauth_key_process_independent_data,
421 ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id),
422 ptrauth_string_discriminator("m_tag.m_tag_cookie")));
423 }
424
425 static void
m_tag_verify_cookie(struct m_tag * tag)426 m_tag_verify_cookie(struct m_tag *tag)
427 {
428 uintptr_t cookie = m_tag_cookie_from_id_and_type(tag);
429 uintptr_t auth_cookie;
430
431 auth_cookie = (uintptr_t) ptrauth_auth_data((void *)(uintptr_t)tag->m_tag_cookie,
432 ptrauth_key_process_independent_data,
433 ptrauth_blend_discriminator((void *)(uintptr_t)(tag->m_tag_type | tag->m_tag_id),
434 ptrauth_string_discriminator("m_tag.m_tag_cookie")));
435 if (cookie != auth_cookie) {
436 panic("verify_m_tag_cookie bad m_tag cookie");
437 }
438 }
439
440 #else /* defined(HAS_APPLE_PAC) */
441
442 void
m_tag_create_cookie(struct m_tag * tag)443 m_tag_create_cookie(struct m_tag *tag)
444 {
445 tag->m_tag_cookie = M_TAG_VALID_PATTERN;
446 }
447
448 static void
m_tag_verify_cookie(struct m_tag * tag)449 m_tag_verify_cookie(struct m_tag *tag)
450 {
451 VERIFY(tag->m_tag_cookie == M_TAG_VALID_PATTERN);
452 }
453
454 #endif /* defined(HAS_APPLE_PAC) */
455
456 #ifdef MB_TAG_MBUF
457 /* Get a packet tag structure along with specified data following. */
458 static struct m_tag *
m_tag_alloc_mbuf(u_int32_t id,u_int16_t type,uint16_t len,int wait)459 m_tag_alloc_mbuf(u_int32_t id, u_int16_t type, uint16_t len, int wait)
460 {
461 struct m_tag *t;
462 void *mb_cl = NULL;
463
464 if (M_TAG_ALIGN(len) + sizeof(struct m_taghdr) <= MLEN) {
465 struct mbuf *m = m_get(wait, MT_TAG);
466 struct m_taghdr *hdr;
467
468 if (m == NULL) {
469 return NULL;
470 }
471 mb_cl = m;
472
473 m->m_flags |= M_TAGHDR;
474
475 hdr = (struct m_taghdr *)(void *)m->m_data;
476 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
477 hdr->mth_refcnt = 1;
478 m->m_len += sizeof(struct m_taghdr);
479 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
480 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
481 m->m_len += M_TAG_ALIGN(len);
482 VERIFY(m->m_len <= MLEN);
483 } else if (len + sizeof(struct m_tag) <= MCLBYTES) {
484 mb_cl = m_mclalloc(wait);
485 t = (struct m_tag *)(void *)mb_cl;
486 } else {
487 t = NULL;
488 }
489
490 if (__improbable(t == NULL)) {
491 return NULL;
492 }
493
494 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
495 M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl);
496 if (len > 0) {
497 bzero(t->m_tag_data, len);
498 }
499 return t;
500 }
501 #endif /* MB_TAG_MBUF */
502
503 static struct m_tag_type_entry *
get_m_tag_type_entry(uint32_t id,uint16_t type,struct m_tag_type_stats ** pmtts)504 get_m_tag_type_entry(uint32_t id, uint16_t type, struct m_tag_type_stats **pmtts)
505 {
506 m_tag_type_entry_ref_t mtte = &m_tag_type_table[KERNEL_TAG_TYPE_NONE];
507
508 if (pmtts != NULL) {
509 *pmtts = &m_tag_type_stats[KERNEL_TAG_TYPE_NONE];
510 }
511
512 if (id == KERNEL_MODULE_TAG_ID) {
513 switch (type) {
514 case KERNEL_TAG_TYPE_DUMMYNET:
515 case KERNEL_TAG_TYPE_IPFILT:
516 case KERNEL_TAG_TYPE_ENCAP:
517 case KERNEL_TAG_TYPE_INET6:
518 case KERNEL_TAG_TYPE_IPSEC:
519 case KERNEL_TAG_TYPE_CFIL_UDP:
520 case KERNEL_TAG_TYPE_PF_REASS:
521 case KERNEL_TAG_TYPE_AQM:
522 case KERNEL_TAG_TYPE_DRVAUX:
523 mtte = &m_tag_type_table[type];
524 if (pmtts != NULL) {
525 *pmtts = &m_tag_type_stats[type];
526 }
527 break;
528 default:
529 #if DEBUG || DEVELOPMENT
530 if (type > 0 && type < KERNEL_TAG_TYPE_COUNT) {
531 panic("get_m_tag_type_entry unexpected m_tag type %u",
532 type);
533 }
534 #endif /* DEBUG || DEVELOPMENT */
535 break;
536 }
537 }
538
539 return mtte;
540 }
541
542 #ifndef MB_TAG_MBUF
543 static struct m_tag *
m_tag_kalloc(uint32_t id,uint16_t type,uint16_t len,int wait,struct m_tag_type_entry * mtte)544 m_tag_kalloc(uint32_t id, uint16_t type, uint16_t len, int wait, struct m_tag_type_entry *mtte)
545 {
546 struct m_tag *tag = NULL;
547
548 tag = mtte->mt_alloc_func(id, type, len, wait);
549
550 if (__probable(tag != NULL)) {
551 VERIFY(IS_P2ALIGNED(tag, sizeof(uint64_t)));
552
553 if (__improbable(tag->m_tag_data == NULL)) {
554 VERIFY(len == 0);
555 } else {
556 VERIFY(len != 0);
557 VERIFY(IS_P2ALIGNED(tag->m_tag_data, sizeof(uint64_t)));
558 }
559 }
560 return tag;
561 }
562
563 static void
m_tag_kfree(struct m_tag * tag,struct m_tag_type_entry * mtte)564 m_tag_kfree(struct m_tag *tag, struct m_tag_type_entry *mtte)
565 {
566 mtte->mt_free_func(tag);
567 }
568 #endif /* MB_TAG_MBUF */
569
570 static struct m_tag *
m_tag_alloc(uint32_t id,uint16_t type,int len,int wait)571 m_tag_alloc(uint32_t id, uint16_t type, int len, int wait)
572 {
573 struct m_tag *tag = NULL;
574 m_tag_type_entry_ref_t mtte = NULL;
575 m_tag_type_stats_ref_t mtts = NULL;
576
577 mtte = get_m_tag_type_entry(id, type, &mtts);
578
579 if (__improbable(len < 0 || len >= MCLBYTES - sizeof(struct m_tag))) {
580 goto done;
581 }
582
583 #ifdef MB_TAG_MBUF
584 tag = m_tag_alloc_mbuf(id, type, (uint16_t)len, wait);
585 #else /* MB_TAG_MBUF */
586 /*
587 * Using Z_NOWAIT could cause retransmission delays when there aren't
588 * many other colocated types in the zone that would prime it. Use
589 * Z_NOPAGEWAIT instead which will only fail to allocate when zalloc
590 * needs to block on the VM for pages.
591 */
592 if (wait & Z_NOWAIT) {
593 wait &= ~Z_NOWAIT;
594 wait |= Z_NOPAGEWAIT;
595 }
596 tag = m_tag_kalloc(id, type, (uint16_t)len, wait, mtte);
597 #endif /* MB_TAG_MBUF */
598
599 done:
600 if (__probable(tag != NULL)) {
601 m_tag_verify_cookie(tag);
602 assert3u(tag->m_tag_id, ==, id);
603 assert3u(tag->m_tag_type, ==, type);
604 assert3u(tag->m_tag_len, ==, len);
605
606 os_atomic_inc(&mtts->mt_alloc_count, relaxed);
607 } else {
608 os_atomic_inc(&mtts->mt_alloc_failed, relaxed);
609 }
610
611 return tag;
612 }
613
614
615 #ifdef MB_TAG_MBUF
616 static struct m_tag *
m_tag_create_mbuf(uint32_t id,uint16_t type,uint16_t len,int wait,struct mbuf * buf)617 m_tag_create_mbuf(uint32_t id, uint16_t type, uint16_t len, int wait, struct mbuf *buf)
618 {
619 struct m_tag *t = NULL;
620 struct m_tag *p;
621 void *mb_cl = NULL;
622
623 if (len + sizeof(struct m_tag) + sizeof(struct m_taghdr) > MLEN) {
624 return m_tag_alloc(id, type, len, wait);
625 }
626
627 /*
628 * We've exhausted all external cases. Now, go through the m_tag
629 * chain and see if we can fit it in any of them.
630 * If not (t == NULL), call m_tag_alloc to store it in a new mbuf.
631 */
632 p = SLIST_FIRST(&buf->m_pkthdr.tags);
633 while (p != NULL) {
634 /* 2KCL m_tag */
635 if (M_TAG_ALIGN(p->m_tag_len) +
636 sizeof(struct m_taghdr) > MLEN) {
637 p = SLIST_NEXT(p, m_tag_link);
638 continue;
639 }
640
641 m_tag_verify_cookie(p);
642
643 struct mbuf *m = p->m_tag_mb_cl;
644 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
645
646 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
647 VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT));
648
649 /* The mbuf can store this m_tag */
650 if (M_TAG_ALIGN(len) <= MLEN - m->m_len) {
651 mb_cl = m;
652 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
653 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
654 hdr->mth_refcnt++;
655 m->m_len += M_TAG_ALIGN(len);
656 VERIFY(m->m_len <= MLEN);
657 break;
658 }
659
660 p = SLIST_NEXT(p, m_tag_link);
661 }
662
663 if (t == NULL) {
664 return m_tag_alloc(id, type, len, wait);
665 }
666
667 M_TAG_INIT(t, id, type, len, (void *)(t + 1), mb_cl);
668 if (len > 0) {
669 bzero(t->m_tag_data, len);
670 }
671 return t;
672 }
673
674 static void
m_tag_free_mbuf(struct m_tag * t)675 m_tag_free_mbuf(struct m_tag *t)
676 {
677 if (__improbable(t == NULL)) {
678 return;
679 }
680
681 if (M_TAG_ALIGN(t->m_tag_len) + sizeof(struct m_taghdr) <= MLEN) {
682 struct mbuf * m = t->m_tag_mb_cl;
683
684 VERIFY(m->m_flags & M_TAGHDR);
685 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
686
687 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
688
689 /* No other tags in this mbuf */
690 if (--hdr->mth_refcnt == 0) {
691 m_free(m);
692 return;
693 }
694
695 /* Pattern-fill the header */
696 u_int64_t *fill_ptr = (u_int64_t *)t;
697 u_int64_t *end_ptr = (u_int64_t *)(t + 1);
698 while (fill_ptr < end_ptr) {
699 *fill_ptr = M_TAG_FREE_PATTERN;
700 fill_ptr++;
701 }
702 } else {
703 m_mclfree((caddr_t)t);
704 }
705 }
706 #endif /* MB_TAG_MBUF */
707
708 struct m_tag *
m_tag_create(uint32_t id,uint16_t type,int len,int wait,struct mbuf * buf)709 m_tag_create(uint32_t id, uint16_t type, int len, int wait, struct mbuf *buf)
710 {
711 #ifdef MB_TAG_MBUF
712 /*
713 * Create and return an m_tag, either by re-using space in a previous tag
714 * or by allocating a new mbuf/cluster
715 */
716 return m_tag_create_mbuf(id, type, (uint16_t)len, wait, buf);
717 #else /* MB_TAG_MBUF */
718 #pragma unused(buf)
719 /*
720 * Each packet tag has its own allocation
721 */
722 return m_tag_alloc(id, type, (uint16_t)len, wait);
723 #endif /* MB_TAG_MBUF */
724 }
725
726 /*
727 * Allocations for external data are known to not have pointers for
728 * most platforms -- for macOS this is not guaranteed
729 */
730 #if XNU_TARGET_OS_OSX
731
732 __typed_allocators_ignore_push
733
734 static inline void * __bidi_indexable
m_tag_data_kalloc(uint16_t len,int wait)735 m_tag_data_kalloc(uint16_t len, int wait)
736 {
737 return kheap_alloc(KHEAP_DEFAULT, len, wait | M_ZERO);
738 }
739
740 static inline void
m_tag_data_free(struct m_tag * tag)741 m_tag_data_free(struct m_tag *tag)
742 {
743 void *tag_data = tag->m_tag_data;
744 size_t tag_len = tag->m_tag_len;
745 kheap_free(KHEAP_DEFAULT, tag_data, tag_len);
746 tag->m_tag_data = NULL;
747 tag->m_tag_len = 0;
748 }
749 __typed_allocators_ignore_pop
750
751 #else /* XNU_TARGET_OS_OSX */
752
753 static inline void * __bidi_indexable
754 m_tag_data_kalloc(uint16_t len, int wait)
755 {
756 return kalloc_data(len, wait | M_ZERO);
757 }
758
759 static inline void
760 m_tag_data_free(struct m_tag *tag)
761 {
762 kfree_data_sized_by(tag->m_tag_data, tag->m_tag_len);
763 }
764
765 #endif /* XNU_TARGET_OS_OSX */
766
767 static struct m_tag *
m_tag_kalloc_external(uint32_t id,uint16_t type,uint16_t len,int wait)768 m_tag_kalloc_external(uint32_t id, uint16_t type, uint16_t len, int wait)
769 {
770 struct m_tag *tag;
771 void *data = NULL;
772
773 tag = kalloc_type(struct m_tag, wait | M_ZERO);
774 if (__improbable(tag == NULL)) {
775 return NULL;
776 }
777
778 if (len > 0) {
779 data = m_tag_data_kalloc(len, wait);
780 if (__improbable(data == NULL)) {
781 kfree_type(struct m_tag, tag);
782 return NULL;
783 }
784 }
785
786 M_TAG_INIT(tag, id, type, len, data, NULL);
787
788 return tag;
789 }
790
791 static void
m_tag_kfree_external(struct m_tag * tag)792 m_tag_kfree_external(struct m_tag *tag)
793 {
794 if (tag->m_tag_data != NULL) {
795 m_tag_data_free(tag);
796 }
797 kfree_type(struct m_tag, tag);
798 }
799
800 /* Free a packet tag. */
801 void
m_tag_free(struct m_tag * tag)802 m_tag_free(struct m_tag *tag)
803 {
804 m_tag_type_entry_ref_t mtte = NULL;
805 m_tag_type_stats_ref_t mtts = NULL;
806
807 if (__improbable(tag == NULL)) {
808 return;
809 }
810
811 m_tag_verify_cookie(tag);
812
813 mtte = get_m_tag_type_entry(tag->m_tag_id, tag->m_tag_type, &mtts);
814
815 #ifdef MB_TAG_MBUF
816 m_tag_free_mbuf(tag);
817 #else /* MB_TAG_MBUF */
818 m_tag_kfree(tag, mtte);
819 #endif /* MB_TAG_MBUF */
820
821 os_atomic_inc(&mtts->mt_free_count, relaxed);
822 }
823
824 void
mbuf_tag_init(void)825 mbuf_tag_init(void)
826 {
827 for (uint16_t type = 0; type < KERNEL_TAG_TYPE_COUNT; type++) {
828 m_tag_type_table[type].mt_type = type;
829 m_tag_type_table[type].mt_len = 0;
830 m_tag_type_table[type].mt_alloc_func = m_tag_kalloc_notsupp;
831 m_tag_type_table[type].mt_free_func = m_tag_kfree_notsupp;
832 }
833 m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_alloc_func = m_tag_kalloc_external;
834 m_tag_type_table[KERNEL_TAG_TYPE_NONE].mt_free_func = m_tag_kfree_external;
835 m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_alloc_func = m_tag_kalloc_external;
836 m_tag_type_table[KERNEL_TAG_TYPE_DRVAUX].mt_free_func = m_tag_kfree_external;
837
838 #if NETWORKING
839 extern void pktsched_register_m_tag(void);
840 pktsched_register_m_tag();
841 #endif /* NETWORKING */
842
843 #if INET
844 extern void ip6_register_m_tag(void);
845 ip6_register_m_tag();
846
847 extern void ipfilter_register_m_tag(void);
848 ipfilter_register_m_tag();
849
850 extern void encap_register_m_tag(void);
851 encap_register_m_tag();
852 #endif /* INET */
853
854 #if IPSEC
855 extern void ipsec_register_m_tag(void);
856 ipsec_register_m_tag();
857 #endif /* IPSEC */
858
859 #if DUMMYNET
860 extern void dummynet_register_m_tag(void);
861 dummynet_register_m_tag();
862 #endif /* DUMMYNET */
863
864 #if PF
865 extern void pf_register_m_tag(void);
866 pf_register_m_tag();
867 #endif /* PF */
868
869 #if CONTENT_FILTER
870 extern void cfil_register_m_tag(void);
871 cfil_register_m_tag();
872 #endif /* CONTENT_FILTER */
873 }
874
875 int
m_register_internal_tag_type(uint16_t type,uint16_t len,m_tag_kalloc_func_t alloc_func,m_tag_kfree_func_t free_func)876 m_register_internal_tag_type(uint16_t type, uint16_t len,
877 m_tag_kalloc_func_t alloc_func, m_tag_kfree_func_t free_func)
878 {
879 int error = 0;
880
881 if (type <= 0 || type >= KERNEL_TAG_TYPE_DRVAUX) {
882 error = EINVAL;
883 goto done;
884 }
885 m_tag_type_table[type].mt_type = type;
886 m_tag_type_table[type].mt_len = len;
887 m_tag_type_table[type].mt_alloc_func = alloc_func;
888 m_tag_type_table[type].mt_free_func = free_func;
889
890 done:
891 return error;
892 }
893
894 /* Prepend a packet tag. */
895 void
m_tag_prepend(struct mbuf * m,struct m_tag * t)896 m_tag_prepend(struct mbuf *m, struct m_tag *t)
897 {
898 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
899 }
900
901 /* Unlink a packet tag. */
902 void
m_tag_unlink(struct mbuf * m,struct m_tag * t)903 m_tag_unlink(struct mbuf *m, struct m_tag *t)
904 {
905 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
906 }
907
908 /* Unlink and free a packet tag. */
909 void
m_tag_delete(struct mbuf * m,struct m_tag * t)910 m_tag_delete(struct mbuf *m, struct m_tag *t)
911 {
912 m_tag_unlink(m, t);
913 m_tag_free(t);
914 }
915
916 /* Unlink and free a packet tag chain, starting from given tag. */
917 void
m_tag_delete_chain(struct mbuf * m)918 m_tag_delete_chain(struct mbuf *m)
919 {
920 struct m_tag *p, *q;
921
922 p = SLIST_FIRST(&m->m_pkthdr.tags);
923 if (p == NULL) {
924 return;
925 }
926
927 while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) {
928 m_tag_delete(m, q);
929 }
930 m_tag_delete(m, p);
931 }
932
933 /* Find a tag, starting from a given position. */
934 struct m_tag *
m_tag_locate(struct mbuf * m,uint32_t id,uint16_t type)935 m_tag_locate(struct mbuf *m, uint32_t id, uint16_t type)
936 {
937 struct m_tag *p;
938
939 VERIFY(m->m_flags & M_PKTHDR);
940
941 p = SLIST_FIRST(&m->m_pkthdr.tags);
942
943 while (p != NULL) {
944 if (p->m_tag_id == id && p->m_tag_type == type) {
945 m_tag_verify_cookie(p);
946 return p;
947 }
948 p = SLIST_NEXT(p, m_tag_link);
949 }
950 return NULL;
951 }
952
953 /* Copy a single tag. */
954 struct m_tag *
m_tag_copy(struct m_tag * t,int how)955 m_tag_copy(struct m_tag *t, int how)
956 {
957 struct m_tag *p;
958
959 VERIFY(t != NULL);
960
961 p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how);
962 if (p == NULL) {
963 return NULL;
964 }
965 bcopy(t->m_tag_data, p->m_tag_data, t->m_tag_len); /* Copy the data */
966 return p;
967 }
968
969 /*
970 * Copy two tag chains. The destination mbuf (to) loses any attached
971 * tags even if the operation fails. This should not be a problem, as
972 * m_tag_copy_chain() is typically called with a newly-allocated
973 * destination mbuf.
974 */
975 int
m_tag_copy_chain(struct mbuf * to,struct mbuf * from,int how)976 m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how)
977 {
978 struct m_tag *p, *t, *tprev = NULL;
979
980 VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR));
981
982 m_tag_delete_chain(to);
983 SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) {
984 m_tag_verify_cookie(p);
985 t = m_tag_copy(p, how);
986 if (t == NULL) {
987 m_tag_delete_chain(to);
988 return 0;
989 }
990 if (tprev == NULL) {
991 SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link);
992 } else {
993 SLIST_INSERT_AFTER(tprev, t, m_tag_link);
994 tprev = t;
995 }
996 }
997 return 1;
998 }
999
1000 /* Initialize dynamic and static tags on an mbuf. */
1001 void
m_tag_init(struct mbuf * m,int all)1002 m_tag_init(struct mbuf *m, int all)
1003 {
1004 VERIFY(m->m_flags & M_PKTHDR);
1005
1006 SLIST_INIT(&m->m_pkthdr.tags);
1007 /*
1008 * If the caller wants to preserve static mbuf tags
1009 * (e.g. m_dup_pkthdr), don't zero them out.
1010 */
1011 if (all) {
1012 bzero(&m->m_pkthdr.builtin_mtag._net_mtag,
1013 sizeof(m->m_pkthdr.builtin_mtag._net_mtag));
1014 }
1015 }
1016
1017 /* Get first tag in chain. */
1018 struct m_tag *
m_tag_first(struct mbuf * m)1019 m_tag_first(struct mbuf *m)
1020 {
1021 VERIFY(m->m_flags & M_PKTHDR);
1022
1023 return SLIST_FIRST(&m->m_pkthdr.tags);
1024 }
1025
1026 /* Get next tag in chain. */
1027 struct m_tag *
m_tag_next(struct mbuf * m,struct m_tag * t)1028 m_tag_next(struct mbuf *m, struct m_tag *t)
1029 {
1030 #pragma unused(m)
1031 VERIFY(t != NULL);
1032
1033 return SLIST_NEXT(t, m_tag_link);
1034 }
1035
1036 int
m_set_traffic_class(struct mbuf * m,mbuf_traffic_class_t tc)1037 m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc)
1038 {
1039 uint32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */
1040
1041 return m_set_service_class(m, m_service_class_from_val(val));
1042 }
1043
1044 mbuf_traffic_class_t
m_get_traffic_class(struct mbuf * m)1045 m_get_traffic_class(struct mbuf *m)
1046 {
1047 return MBUF_SC2TC(m_get_service_class(m));
1048 }
1049
1050 int
m_set_service_class(struct mbuf * m,mbuf_svc_class_t sc)1051 m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc)
1052 {
1053 int error = 0;
1054
1055 VERIFY(m->m_flags & M_PKTHDR);
1056
1057 if (MBUF_VALID_SC(sc)) {
1058 m->m_pkthdr.pkt_svc = sc;
1059 } else {
1060 error = EINVAL;
1061 }
1062
1063 return error;
1064 }
1065
1066 mbuf_svc_class_t
m_get_service_class(struct mbuf * m)1067 m_get_service_class(struct mbuf *m)
1068 {
1069 mbuf_svc_class_t sc;
1070
1071 VERIFY(m->m_flags & M_PKTHDR);
1072
1073 if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) {
1074 sc = m->m_pkthdr.pkt_svc;
1075 } else {
1076 sc = MBUF_SC_BE;
1077 }
1078
1079 return sc;
1080 }
1081
1082 mbuf_svc_class_t
m_service_class_from_idx(uint32_t i)1083 m_service_class_from_idx(uint32_t i)
1084 {
1085 mbuf_svc_class_t sc = MBUF_SC_BE;
1086
1087 switch (i) {
1088 case SCIDX_BK_SYS:
1089 return MBUF_SC_BK_SYS;
1090
1091 case SCIDX_BK:
1092 return MBUF_SC_BK;
1093
1094 case SCIDX_BE:
1095 return MBUF_SC_BE;
1096
1097 case SCIDX_RD:
1098 return MBUF_SC_RD;
1099
1100 case SCIDX_OAM:
1101 return MBUF_SC_OAM;
1102
1103 case SCIDX_AV:
1104 return MBUF_SC_AV;
1105
1106 case SCIDX_RV:
1107 return MBUF_SC_RV;
1108
1109 case SCIDX_VI:
1110 return MBUF_SC_VI;
1111
1112 case SCIDX_VO:
1113 return MBUF_SC_VO;
1114
1115 case SCIDX_CTL:
1116 return MBUF_SC_CTL;
1117
1118 default:
1119 break;
1120 }
1121
1122 VERIFY(0);
1123 /* NOTREACHED */
1124 return sc;
1125 }
1126
1127 mbuf_svc_class_t
m_service_class_from_val(uint32_t v)1128 m_service_class_from_val(uint32_t v)
1129 {
1130 mbuf_svc_class_t sc = MBUF_SC_BE;
1131
1132 switch (v) {
1133 case SCVAL_BK_SYS:
1134 return MBUF_SC_BK_SYS;
1135
1136 case SCVAL_BK:
1137 return MBUF_SC_BK;
1138
1139 case SCVAL_BE:
1140 return MBUF_SC_BE;
1141
1142 case SCVAL_RD:
1143 return MBUF_SC_RD;
1144
1145 case SCVAL_OAM:
1146 return MBUF_SC_OAM;
1147
1148 case SCVAL_AV:
1149 return MBUF_SC_AV;
1150
1151 case SCVAL_RV:
1152 return MBUF_SC_RV;
1153
1154 case SCVAL_VI:
1155 return MBUF_SC_VI;
1156
1157 case SCVAL_VO:
1158 return MBUF_SC_VO;
1159
1160 case SCVAL_CTL:
1161 return MBUF_SC_CTL;
1162
1163 default:
1164 break;
1165 }
1166
1167 VERIFY(0);
1168 /* NOTREACHED */
1169 return sc;
1170 }
1171
1172 uint16_t
m_adj_sum16(struct mbuf * m,uint32_t start,uint32_t dataoff,uint32_t datalen,uint32_t sum)1173 m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff,
1174 uint32_t datalen, uint32_t sum)
1175 {
1176 uint32_t total_sub = 0; /* total to subtract */
1177 uint32_t mlen = m_pktlen(m); /* frame length */
1178 uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */
1179 int len;
1180
1181 ASSERT(bytes <= mlen);
1182
1183 /*
1184 * Take care of excluding (len > 0) or including (len < 0)
1185 * extraneous octets at the beginning of the packet, taking
1186 * into account the start offset.
1187 */
1188 len = (dataoff - start);
1189 if (len > 0) {
1190 total_sub = m_sum16(m, start, len);
1191 } else if (len < 0) {
1192 sum += m_sum16(m, dataoff, -len);
1193 }
1194
1195 /*
1196 * Take care of excluding any postpended extraneous octets.
1197 */
1198 len = (mlen - bytes);
1199 if (len > 0) {
1200 struct mbuf *m0 = m;
1201 uint32_t extra = m_sum16(m, bytes, len);
1202 uint32_t off = bytes, off0 = off;
1203
1204 while (off > 0) {
1205 if (__improbable(m == NULL)) {
1206 panic("%s: invalid mbuf chain %p [off %u, "
1207 "len %u]", __func__, m0, off0, len);
1208 /* NOTREACHED */
1209 }
1210 if (off < m->m_len) {
1211 break;
1212 }
1213 off -= m->m_len;
1214 m = m->m_next;
1215 }
1216
1217 /* if we started on odd-alignment, swap the value */
1218 if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) {
1219 total_sub += ((extra << 8) & 0xffff) | (extra >> 8);
1220 } else {
1221 total_sub += extra;
1222 }
1223
1224 total_sub = (total_sub >> 16) + (total_sub & 0xffff);
1225 }
1226
1227 /*
1228 * 1's complement subtract any extraneous octets.
1229 */
1230 if (total_sub != 0) {
1231 if (total_sub >= sum) {
1232 sum = ~(total_sub - sum) & 0xffff;
1233 } else {
1234 sum -= total_sub;
1235 }
1236 }
1237
1238 /* fold 32-bit to 16-bit */
1239 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
1240 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
1241 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
1242
1243 return sum & 0xffff;
1244 }
1245
1246 uint16_t
m_sum16(struct mbuf * m,uint32_t off,uint32_t len)1247 m_sum16(struct mbuf *m, uint32_t off, uint32_t len)
1248 {
1249 int mlen;
1250
1251 /*
1252 * Sanity check
1253 *
1254 * Use m_length2() instead of m_length(), as we cannot rely on
1255 * the caller setting m_pkthdr.len correctly, if the mbuf is
1256 * a M_PKTHDR one.
1257 */
1258 if ((mlen = m_length2(m, NULL)) < (off + len)) {
1259 panic("%s: mbuf %p len (%d) < off+len (%d+%d)", __func__,
1260 m, mlen, off, len);
1261 /* NOTREACHED */
1262 }
1263
1264 return (uint16_t)os_cpu_in_cksum_mbuf(m, len, off, 0);
1265 }
1266
1267 /*
1268 * Write packet tx_time to the mbuf's meta data.
1269 */
1270 void
mbuf_set_tx_time(struct mbuf * m,uint64_t tx_time)1271 mbuf_set_tx_time(struct mbuf *m, uint64_t tx_time)
1272 {
1273 struct m_tag *tag = NULL;
1274 tag = m_tag_create(KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_AQM,
1275 sizeof(uint64_t), M_WAITOK, m);
1276 if (tag != NULL) {
1277 m_tag_prepend(m, tag);
1278 *(uint64_t *)tag->m_tag_data = tx_time;
1279 }
1280 }
1281
1282
1283 static int
sysctl_mb_tag_stats(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)1284 sysctl_mb_tag_stats(__unused struct sysctl_oid *oidp,
1285 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
1286 {
1287 int error = 0;
1288
1289 if (req->oldptr == USER_ADDR_NULL) {
1290 req->oldidx = KERNEL_TAG_TYPE_COUNT * sizeof(struct m_tag_stats);
1291 return 0;
1292 }
1293 if (req->newptr != USER_ADDR_NULL) {
1294 return EPERM;
1295 }
1296
1297 for (uint16_t i = 0; i < KERNEL_TAG_TYPE_COUNT; i++) {
1298 struct m_tag_stats m_tag_stats = {};
1299
1300 m_tag_stats.mts_id = KERNEL_MODULE_TAG_ID;
1301 m_tag_stats.mts_type = i;
1302 m_tag_stats.mts_len = m_tag_type_table[i].mt_len;
1303 m_tag_stats.mts_alloc_count = m_tag_type_stats[i].mt_alloc_count;
1304 m_tag_stats.mts_alloc_failed = m_tag_type_stats[i].mt_alloc_failed;
1305 m_tag_stats.mts_free_count = m_tag_type_stats[i].mt_free_count;
1306
1307 error = SYSCTL_OUT(req, &m_tag_stats, sizeof(struct m_tag_stats));
1308 }
1309
1310 return error;
1311 }
1312
1313 SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_stats,
1314 CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED, NULL, 0,
1315 sysctl_mb_tag_stats, "S,m_tag_stats", "");
1316
1317 #if DEBUG || DEVELOPMENT
1318
1319 struct m_tag_test_entry {
1320 bool mtte_test_id;
1321 bool mtte_alloc_must_fail;
1322 uint16_t mtte_type;
1323 int mtte_len;
1324 };
1325
1326 struct m_tag_test_entry
1327 m_tag_test_table[] = {
1328 {
1329 .mtte_test_id = false,
1330 .mtte_alloc_must_fail = false,
1331 .mtte_type = KERNEL_TAG_TYPE_DUMMYNET,
1332 .mtte_len = 0,
1333 },
1334 {
1335 .mtte_test_id = false,
1336 .mtte_alloc_must_fail = false,
1337 .mtte_type = KERNEL_TAG_TYPE_IPFILT,
1338 .mtte_len = 0,
1339 },
1340 {
1341 .mtte_test_id = false,
1342 .mtte_alloc_must_fail = false,
1343 .mtte_type = KERNEL_TAG_TYPE_ENCAP,
1344 .mtte_len = 0,
1345 },
1346 {
1347 .mtte_test_id = false,
1348 .mtte_alloc_must_fail = false,
1349 .mtte_type = KERNEL_TAG_TYPE_INET6,
1350 .mtte_len = 0,
1351 },
1352 {
1353 .mtte_test_id = false,
1354 .mtte_alloc_must_fail = false,
1355 .mtte_type = KERNEL_TAG_TYPE_IPSEC,
1356 .mtte_len = 0,
1357 },
1358 {
1359 .mtte_test_id = false,
1360 .mtte_alloc_must_fail = false,
1361 .mtte_type = KERNEL_TAG_TYPE_CFIL_UDP,
1362 .mtte_len = 0,
1363 },
1364 {
1365 .mtte_test_id = false,
1366 .mtte_alloc_must_fail = false,
1367 .mtte_type = KERNEL_TAG_TYPE_PF_REASS,
1368 .mtte_len = 0,
1369 },
1370 {
1371 .mtte_test_id = false,
1372 .mtte_alloc_must_fail = false,
1373 .mtte_type = KERNEL_TAG_TYPE_AQM,
1374 .mtte_len = 0,
1375 },
1376 {
1377 .mtte_test_id = false,
1378 .mtte_alloc_must_fail = false,
1379 .mtte_type = KERNEL_TAG_TYPE_DRVAUX,
1380 .mtte_len = 0,
1381 },
1382
1383 {
1384 .mtte_test_id = false,
1385 .mtte_alloc_must_fail = false,
1386 .mtte_type = 0,
1387 .mtte_len = MLEN,
1388 },
1389 {
1390 .mtte_test_id = false,
1391 .mtte_alloc_must_fail = false,
1392 .mtte_type = KERNEL_TAG_TYPE_COUNT,
1393 .mtte_len = MLEN,
1394 },
1395 {
1396 .mtte_test_id = false,
1397 .mtte_alloc_must_fail = true,
1398 .mtte_type = 0,
1399 .mtte_len = MCLBYTES,
1400 },
1401 {
1402 .mtte_test_id = false,
1403 .mtte_alloc_must_fail = true,
1404 .mtte_type = KERNEL_TAG_TYPE_COUNT,
1405 .mtte_len = MCLBYTES,
1406 },
1407
1408 {
1409 .mtte_test_id = true,
1410 .mtte_alloc_must_fail = false,
1411 .mtte_type = 0,
1412 .mtte_len = 0,
1413 },
1414 {
1415 .mtte_test_id = true,
1416 .mtte_alloc_must_fail = false,
1417 .mtte_type = 0,
1418 .mtte_len = MLEN,
1419 },
1420 {
1421 .mtte_test_id = true,
1422 .mtte_alloc_must_fail = true,
1423 .mtte_type = 0,
1424 .mtte_len = -1,
1425 },
1426 {
1427 .mtte_test_id = true,
1428 .mtte_alloc_must_fail = true,
1429 .mtte_type = 0,
1430 .mtte_len = MCLBYTES,
1431 },
1432 };
1433
1434 #define M_TAG_TEST_TABLE_COUNT (sizeof(m_tag_test_table) / sizeof(struct m_tag_test_entry))
1435
1436 #define M_TAG_TEST_ID "com.apple.test.m_tag"
1437
1438 static int
do_m_tag_test(mbuf_tag_id_t test_tag_id)1439 do_m_tag_test(mbuf_tag_id_t test_tag_id)
1440 {
1441 int error = 0;
1442 struct mbuf *m = NULL;
1443
1444 m = m_getpacket();
1445 if (m == NULL) {
1446 os_log_error(OS_LOG_DEFAULT, "%s: m_getpacket failed", __func__);
1447 error = ENOMEM;
1448 goto done;
1449 }
1450
1451 for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) {
1452 struct m_tag_test_entry *entry = &m_tag_test_table[i];
1453 struct m_tag *tag = NULL;
1454 uint32_t id = test_tag_id;
1455 int len = entry->mtte_len;
1456 uint16_t type = entry->mtte_type;
1457
1458 if (entry->mtte_test_id == false) {
1459 id = KERNEL_MODULE_TAG_ID;
1460 switch (type) {
1461 case KERNEL_TAG_TYPE_DUMMYNET:
1462 case KERNEL_TAG_TYPE_IPFILT:
1463 case KERNEL_TAG_TYPE_ENCAP:
1464 case KERNEL_TAG_TYPE_INET6:
1465 case KERNEL_TAG_TYPE_IPSEC:
1466 case KERNEL_TAG_TYPE_CFIL_UDP:
1467 case KERNEL_TAG_TYPE_PF_REASS:
1468 case KERNEL_TAG_TYPE_AQM:
1469 /* subsystems that use mbuf tags are optional */
1470 if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) {
1471 continue;
1472 }
1473 len = m_tag_type_table[type].mt_len;
1474 if (entry->mtte_alloc_must_fail == true) {
1475 os_log_error(OS_LOG_DEFAULT,
1476 "%s: FAIL m_tag_create(%u, %u, %u) must not fail",
1477 __func__, id, type, len);
1478 error = EINVAL;
1479 goto done;
1480 }
1481 break;
1482 default:
1483 break;
1484 }
1485 }
1486 tag = m_tag_create(id, type, len, M_WAIT, m);
1487 if (tag == NULL) {
1488 if (entry->mtte_alloc_must_fail == false) {
1489 os_log_error(OS_LOG_DEFAULT,
1490 "%s: FAIL m_tag_create(%u, %u, %u) unexpected failure",
1491 __func__, id, type, len);
1492 error = ENOMEM;
1493 goto done;
1494 } else {
1495 os_log(OS_LOG_DEFAULT,
1496 "%s: PASS m_tag_create(%u, %u, %u) expected failure",
1497 __func__, id, type, len);
1498 }
1499 } else {
1500 if (entry->mtte_alloc_must_fail == true) {
1501 os_log_error(OS_LOG_DEFAULT,
1502 "%s: FAIL m_tag_create(%u, %u, %u) unexpected success",
1503 __func__, id, type, len);
1504 error = EINVAL;
1505 goto done;
1506 } else {
1507 os_log(OS_LOG_DEFAULT,
1508 "%s: PASS m_tag_create(%u, %u, %u) expected success",
1509 __func__, id, type, len);
1510 }
1511 m_tag_prepend(m, tag);
1512 }
1513 }
1514 done:
1515 if (m != NULL) {
1516 m_freem(m);
1517 }
1518 os_log_error(OS_LOG_DEFAULT,
1519 "%s: %s error %d",
1520 __func__, error == 0 ? "PASS" : "FAIL", error);
1521 return error;
1522 }
1523
1524 static int
do_test_m_tag_unlink(mbuf_tag_id_t test_tag_id)1525 do_test_m_tag_unlink(mbuf_tag_id_t test_tag_id)
1526 {
1527 struct mbuf *m = NULL;
1528 int error = 0;
1529
1530 m = m_gethdr(M_WAITOK, MT_DATA);
1531 if (m == NULL) {
1532 error = ENOMEM;
1533 goto done;
1534 }
1535 for (int i = 0; i < M_TAG_TEST_TABLE_COUNT; i++) {
1536 struct m_tag_test_entry *entry = &m_tag_test_table[i];
1537 struct m_tag *tag = NULL;
1538 uint32_t id = test_tag_id;
1539 int len = entry->mtte_len;
1540 uint16_t type = entry->mtte_type;
1541
1542 if (entry->mtte_alloc_must_fail == true) {
1543 continue;
1544 }
1545
1546 if (entry->mtte_test_id == false) {
1547 id = KERNEL_MODULE_TAG_ID;
1548 switch (type) {
1549 case KERNEL_TAG_TYPE_DUMMYNET:
1550 case KERNEL_TAG_TYPE_IPFILT:
1551 case KERNEL_TAG_TYPE_ENCAP:
1552 case KERNEL_TAG_TYPE_INET6:
1553 case KERNEL_TAG_TYPE_IPSEC:
1554 case KERNEL_TAG_TYPE_CFIL_UDP:
1555 case KERNEL_TAG_TYPE_PF_REASS:
1556 case KERNEL_TAG_TYPE_AQM:
1557 /* subsystems that use mbuf tags are optional */
1558 if (m_tag_type_table[type].mt_alloc_func == m_tag_kalloc_notsupp) {
1559 continue;
1560 }
1561 len = m_tag_type_table[type].mt_len;
1562 break;
1563 default:
1564 continue;
1565 }
1566 }
1567 tag = m_tag_create(id, type, len, M_WAIT, m);
1568 if (tag == NULL) {
1569 os_log_error(OS_LOG_DEFAULT,
1570 "%s: FAIL m_tag_create(%u, %u, %u) failure",
1571 __func__, id, type, len);
1572 error = ENOMEM;
1573 goto done;
1574 } else {
1575 os_log_error(OS_LOG_DEFAULT,
1576 "%s: PASS m_tag_create(%u, %u, %u) success",
1577 __func__, id, type, len);
1578 m_tag_prepend(m, tag);
1579 }
1580 }
1581
1582 struct m_tag *cfil_tag = m_tag_locate(m, KERNEL_MODULE_TAG_ID, KERNEL_TAG_TYPE_CFIL_UDP);
1583 if (cfil_tag == NULL) {
1584 os_log_error(OS_LOG_DEFAULT,
1585 "%s: FAIL m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) failure",
1586 __func__);
1587 error = EINVAL;
1588 goto done;
1589 } else {
1590 os_log_error(OS_LOG_DEFAULT,
1591 "%s: PASS m_tag_locate(KERNEL_TAG_TYPE_CFIL_UDP) success",
1592 __func__);
1593 }
1594
1595 /*
1596 * Unlink the mbuf tag, free the mbuf and finally free the mbuf tag
1597 */
1598 m_tag_unlink(m, cfil_tag);
1599
1600 m_freem(m);
1601 m = NULL;
1602
1603 m_tag_free(cfil_tag);
1604
1605 done:
1606 if (m != NULL) {
1607 m_freem(m);
1608 }
1609 os_log_error(OS_LOG_DEFAULT,
1610 "%s: %s error %d",
1611 __func__, error == 0 ? "PASS" : "FAIL", error);
1612 return error;
1613 }
1614
1615 static int
sysctl_mb_tag_test(__unused struct sysctl_oid * oidp,__unused void * arg1,__unused int arg2,struct sysctl_req * req)1616 sysctl_mb_tag_test(__unused struct sysctl_oid *oidp,
1617 __unused void *arg1, __unused int arg2, struct sysctl_req *req)
1618 {
1619 int error;
1620 int newvalue;
1621 int changed;
1622 int value = 0;
1623 mbuf_tag_id_t test_tag_id;
1624
1625 if ((error = sysctl_io_number(req, value, sizeof(int),
1626 &newvalue, &changed)) != 0) {
1627 goto done;
1628 }
1629 if (!changed && newvalue == value) {
1630 goto done;
1631 }
1632 error = mbuf_tag_id_find(M_TAG_TEST_ID, &test_tag_id);
1633 if (error != 0) {
1634 os_log_error(OS_LOG_DEFAULT, "%s: mbuf_tag_id_find failed error %d",
1635 __func__, error);
1636 goto done;
1637 }
1638 error = do_m_tag_test(test_tag_id);
1639 if (error != 0) {
1640 goto done;
1641 }
1642 error = do_test_m_tag_unlink(test_tag_id);
1643 if (error != 0) {
1644 goto done;
1645 }
1646 done:
1647 return error;
1648 }
1649
1650 SYSCTL_PROC(_kern_ipc, OID_AUTO, mb_tag_test,
1651 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, NULL, 0,
1652 sysctl_mb_tag_test, "I", "mbuf test");
1653
1654 #endif /* DEBUG || DEVELOPMENT */
1655