1 /*
2 * Copyright (c) 2000-2018 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* $NetBSD: uipc_mbuf.c,v 1.40 1999/04/01 00:23:25 thorpej Exp $ */
29
30 /*
31 * Copyright (C) 1999 WIDE Project.
32 * All rights reserved.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. Neither the name of the project nor the names of its contributors
43 * may be used to endorse or promote products derived from this software
44 * without specific prior written permission.
45 *
46 * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND
47 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
48 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
49 * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE
50 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
51 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
52 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
53 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
54 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
55 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
56 * SUCH DAMAGE.
57 */
58
59 /*
60 * Copyright (c) 1982, 1986, 1988, 1991, 1993
61 * The Regents of the University of California. All rights reserved.
62 *
63 * Redistribution and use in source and binary forms, with or without
64 * modification, are permitted provided that the following conditions
65 * are met:
66 * 1. Redistributions of source code must retain the above copyright
67 * notice, this list of conditions and the following disclaimer.
68 * 2. Redistributions in binary form must reproduce the above copyright
69 * notice, this list of conditions and the following disclaimer in the
70 * documentation and/or other materials provided with the distribution.
71 * 3. All advertising materials mentioning features or use of this software
72 * must display the following acknowledgement:
73 * This product includes software developed by the University of
74 * California, Berkeley and its contributors.
75 * 4. Neither the name of the University nor the names of its contributors
76 * may be used to endorse or promote products derived from this software
77 * without specific prior written permission.
78 *
79 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
80 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
81 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
82 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
83 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
84 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
85 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
86 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
87 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
88 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
89 * SUCH DAMAGE.
90 *
91 * @(#)uipc_mbuf.c 8.4 (Berkeley) 2/14/95
92 */
93 /*
94 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
95 * support for mandatory and extensible security protections. This notice
96 * is included in support of clause 2.2 (b) of the Apple Public License,
97 * Version 2.0.
98 */
99
100
101 /*#define PULLDOWN_DEBUG*/
102
103 #include <sys/param.h>
104 #include <sys/systm.h>
105 #include <sys/proc_internal.h>
106 #include <sys/malloc.h>
107 #include <sys/mbuf.h>
108 #include <sys/mcache.h>
109 #include <netinet/in.h>
110 #include <netinet/ip_var.h>
111 #include <netinet/ip6.h>
112 #include <netinet6/ip6_var.h>
113
114 /*
115 * ensure that [off, off + len) is contiguous on the mbuf chain "m".
116 * packet chain before "off" is kept untouched.
117 * if offp == NULL, the target will start at <retval, 0> on resulting chain.
118 * if offp != NULL, the target will start at <retval, *offp> on resulting chain.
119 *
120 * on error return (NULL return value), original "m" will be freed.
121 *
122 * XXX M_TRAILINGSPACE/M_LEADINGSPACE on shared cluster (sharedcluster)
123 */
124 struct mbuf *
m_pulldown(struct mbuf * m,int off,int len,int * offp)125 m_pulldown(struct mbuf *m, int off, int len, int *offp)
126 {
127 struct mbuf *n = NULL, *o = NULL;
128 int hlen = 0, tlen = 0, olen = 0;
129 int sharedcluster = 0;
130
131 /* check invalid arguments. */
132 VERIFY(len >= 0 && off >= 0);
133
134 if (m == NULL) {
135 panic("m == NULL in m_pulldown()");
136 }
137 if (len > MCLBYTES) {
138 m_freem(m);
139 return NULL; /* impossible */
140 }
141 int tmp_len = 0;
142 if (os_add_overflow(off, len, &tmp_len)) {
143 m_free(m);
144 return NULL;
145 }
146
147 #ifdef PULLDOWN_DEBUG
148 {
149 struct mbuf *t;
150 printf("before:");
151 for (t = m; t; t = t->m_next) {
152 printf(" %d", t->m_len);
153 }
154 printf("\n");
155 }
156 #endif
157 n = m;
158
159 /*
160 * Iterate and make n point to the mbuf
161 * within which the first byte at length
162 * offset is contained from the start of
163 * mbuf chain.
164 */
165 while (n != NULL && off > 0) {
166 if (n->m_len > off) {
167 break;
168 }
169 off -= n->m_len;
170 n = n->m_next;
171 }
172
173 /* be sure to point non-empty mbuf */
174 while (n != NULL && n->m_len == 0) {
175 n = n->m_next;
176 }
177
178 if (!n) {
179 m_freem(m);
180 return NULL; /* mbuf chain too short */
181 }
182
183 /*
184 * the target data is on <n, off>.
185 * if we got enough data on the mbuf "n", we're done.
186 *
187 * It should be noted, that we should only do this either
188 * when offset is 0, i.e. data is pointing to the start
189 * or when the caller specifies an out argument to get
190 * the offset value in the mbuf to work with data pointer
191 * correctly.
192 *
193 * If offset is not 0 and caller did not provide out-argument
194 * to get offset, we should split the mbuf even when the length
195 * is contained in current mbuf.
196 */
197 if ((off == 0 || offp) && len <= n->m_len - off) {
198 goto ok;
199 }
200
201 /*
202 * when len <= n->m_len - off and off != 0, it is a special case.
203 * len bytes from <n, off> sits in single mbuf, but the caller does
204 * not like the starting position (off).
205 * chop the current mbuf into two pieces, set off to 0.
206 */
207 if (len <= n->m_len - off) {
208 o = m_copym(n, off, n->m_len - off, M_DONTWAIT);
209 if (o == NULL) {
210 m_freem(m);
211 return NULL; /* ENOBUFS */
212 }
213 n->m_len = off;
214 o->m_next = n->m_next;
215 n->m_next = o;
216 n = n->m_next;
217 off = 0;
218 goto ok;
219 }
220
221 /*
222 * we need to take hlen from <n, off> and tlen from <n->m_next, 0>,
223 * and construct contiguous mbuf with m_len == len.
224 * note that hlen + tlen == len, and tlen > 0.
225 *
226 * Read these variables as head length and tail length
227 */
228 hlen = n->m_len - off;
229 tlen = len - hlen;
230
231 /*
232 * ensure that we have enough trailing data on mbuf chain.
233 * if not, we can do nothing about the chain.
234 */
235 olen = 0;
236 for (o = n->m_next; o != NULL; o = o->m_next) {
237 olen += o->m_len;
238 }
239 if (hlen + olen < len) {
240 m_freem(m);
241 return NULL; /* mbuf chain too short */
242 }
243
244 /*
245 * easy cases first.
246 * we need to use m_copydata() to get data from <n->m_next, 0>.
247 */
248 if ((n->m_flags & M_EXT) == 0) {
249 sharedcluster = 0;
250 } else {
251 if (m_get_ext_free(n) != NULL) {
252 sharedcluster = 1;
253 } else if (m_mclhasreference(n)) {
254 sharedcluster = 1;
255 } else {
256 sharedcluster = 0;
257 }
258 }
259
260 /*
261 * If we have enough space left in current mbuf to accomodate
262 * tail length, copy tail length worth of data starting with next mbuf
263 * and adjust the length of next one accordingly.
264 */
265 if ((off == 0 || offp) && M_TRAILINGSPACE(n) >= tlen
266 && !sharedcluster) {
267 m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len);
268 n->m_len += tlen;
269 m_adj(n->m_next, tlen);
270 goto ok;
271 }
272
273 /*
274 * If have enough leading space in next mbuf to accomodate head length
275 * of current mbuf, and total resulting length of next mbuf is greater
276 * than or equal to requested len bytes, then just copy hlen from
277 * current to the next one and adjust sizes accordingly.
278 */
279 if ((off == 0 || offp) && M_LEADINGSPACE(n->m_next) >= hlen &&
280 (n->m_next->m_len + hlen) >= len && !sharedcluster) {
281 n->m_next->m_data -= hlen;
282 n->m_next->m_len += hlen;
283 bcopy(mtod(n, caddr_t) + off, mtod(n->m_next, caddr_t), hlen);
284 n->m_len -= hlen;
285 n = n->m_next;
286 off = 0;
287 goto ok;
288 }
289
290 /*
291 * now, we need to do the hard way. don't m_copy as there's no room
292 * on both end.
293 */
294 MGET(o, M_DONTWAIT, m->m_type);
295 if (o == NULL) {
296 m_freem(m);
297 return NULL; /* ENOBUFS */
298 }
299 if (len > MHLEN) { /* use MHLEN just for safety */
300 MCLGET(o, M_DONTWAIT);
301 if ((o->m_flags & M_EXT) == 0) {
302 m_freem(m);
303 m_free(o);
304 return NULL; /* ENOBUFS */
305 }
306 }
307 /* get hlen from <n, off> into <o, 0> */
308 o->m_len = hlen;
309 bcopy(mtod(n, caddr_t) + off, mtod(o, caddr_t), hlen);
310 n->m_len -= hlen;
311 /* get tlen from <n->m_next, 0> into <o, hlen> */
312 m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len);
313 o->m_len += tlen;
314 m_adj(n->m_next, tlen);
315 o->m_next = n->m_next;
316 n->m_next = o;
317 n = o;
318 off = 0;
319
320 ok:
321 #ifdef PULLDOWN_DEBUG
322 {
323 struct mbuf *t;
324 printf("after:");
325 for (t = m; t; t = t->m_next) {
326 printf("%c%d", t == n ? '*' : ' ', t->m_len);
327 }
328 printf(" (off=%d)\n", off);
329 }
330 #endif
331 if (offp) {
332 *offp = off;
333 }
334 return n;
335 }
336
337 /*
338 * Create and return an m_tag, either by re-using space in a previous tag
339 * or by allocating a new mbuf/cluster
340 */
341 struct m_tag *
m_tag_create(u_int32_t id,u_int16_t type,int len,int wait,struct mbuf * buf)342 m_tag_create(u_int32_t id, u_int16_t type, int len, int wait, struct mbuf *buf)
343 {
344 struct m_tag *t = NULL;
345 struct m_tag *p;
346
347 if (len < 0) {
348 return NULL;
349 }
350
351 if (len + sizeof(struct m_tag) + sizeof(struct m_taghdr) > MLEN) {
352 return m_tag_alloc(id, type, len, wait);
353 }
354
355 /*
356 * We've exhausted all external cases. Now, go through the m_tag
357 * chain and see if we can fit it in any of them.
358 * If not (t == NULL), call m_tag_alloc to store it in a new mbuf.
359 */
360 p = SLIST_FIRST(&buf->m_pkthdr.tags);
361 while (p != NULL) {
362 /* 2KCL m_tag */
363 if (M_TAG_ALIGN(p->m_tag_len) +
364 sizeof(struct m_taghdr) > MLEN) {
365 p = SLIST_NEXT(p, m_tag_link);
366 continue;
367 }
368
369 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
370
371 struct mbuf *m = m_dtom(p);
372 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
373
374 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
375 VERIFY(m->m_flags & M_TAGHDR && !(m->m_flags & M_EXT));
376
377 /* The mbuf can store this m_tag */
378 if (M_TAG_ALIGN(len) <= MLEN - m->m_len) {
379 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
380 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
381 hdr->refcnt++;
382 m->m_len += M_TAG_ALIGN(len);
383 VERIFY(m->m_len <= MLEN);
384 break;
385 }
386
387 p = SLIST_NEXT(p, m_tag_link);
388 }
389
390 if (t == NULL) {
391 return m_tag_alloc(id, type, len, wait);
392 }
393
394 t->m_tag_cookie = M_TAG_VALID_PATTERN;
395 t->m_tag_type = type;
396 t->m_tag_len = (uint16_t)len;
397 t->m_tag_id = id;
398 if (len > 0) {
399 bzero(t + 1, len);
400 }
401 return t;
402 }
403
404 /* Get a packet tag structure along with specified data following. */
405 struct m_tag *
m_tag_alloc(u_int32_t id,u_int16_t type,int len,int wait)406 m_tag_alloc(u_int32_t id, u_int16_t type, int len, int wait)
407 {
408 struct m_tag *t;
409
410 if (len < 0) {
411 return NULL;
412 }
413
414 if (M_TAG_ALIGN(len) + sizeof(struct m_taghdr) <= MLEN) {
415 struct mbuf *m = m_get(wait, MT_TAG);
416 struct m_taghdr *hdr;
417
418 if (m == NULL) {
419 return NULL;
420 }
421
422 m->m_flags |= M_TAGHDR;
423
424 hdr = (struct m_taghdr *)(void *)m->m_data;
425 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
426 hdr->refcnt = 1;
427 m->m_len += sizeof(struct m_taghdr);
428 t = (struct m_tag *)(void *)(m->m_data + m->m_len);
429 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
430 m->m_len += M_TAG_ALIGN(len);
431 VERIFY(m->m_len <= MLEN);
432 } else if (len + sizeof(struct m_tag) <= MCLBYTES) {
433 t = (struct m_tag *)(void *)m_mclalloc(wait);
434 } else {
435 t = NULL;
436 }
437
438 if (t == NULL) {
439 return NULL;
440 }
441
442 VERIFY(IS_P2ALIGNED(t, sizeof(u_int64_t)));
443 t->m_tag_cookie = M_TAG_VALID_PATTERN;
444 t->m_tag_type = type;
445 t->m_tag_len = (uint16_t)len;
446 t->m_tag_id = id;
447 if (len > 0) {
448 bzero(t + 1, len);
449 }
450 return t;
451 }
452
453
454 /* Free a packet tag. */
455 void
m_tag_free(struct m_tag * t)456 m_tag_free(struct m_tag *t)
457 {
458 if (t == NULL) {
459 return;
460 }
461
462 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
463
464 if (M_TAG_ALIGN(t->m_tag_len) + sizeof(struct m_taghdr) <= MLEN) {
465 struct mbuf * m = m_dtom(t);
466 VERIFY(m->m_flags & M_TAGHDR);
467 struct m_taghdr *hdr = (struct m_taghdr *)(void *)m->m_data;
468
469 VERIFY(IS_P2ALIGNED(hdr + 1, sizeof(u_int64_t)));
470
471 /* No other tags in this mbuf */
472 if (--hdr->refcnt == 0) {
473 m_free(m);
474 return;
475 }
476
477 /* Pattern-fill the header */
478 u_int64_t *fill_ptr = (u_int64_t *)t;
479 u_int64_t *end_ptr = (u_int64_t *)(t + 1);
480 while (fill_ptr < end_ptr) {
481 *fill_ptr = M_TAG_FREE_PATTERN;
482 fill_ptr++;
483 }
484 } else {
485 m_mclfree((caddr_t)t);
486 }
487 }
488
489 /* Prepend a packet tag. */
490 void
m_tag_prepend(struct mbuf * m,struct m_tag * t)491 m_tag_prepend(struct mbuf *m, struct m_tag *t)
492 {
493 VERIFY(m != NULL && t != NULL);
494
495 SLIST_INSERT_HEAD(&m->m_pkthdr.tags, t, m_tag_link);
496 }
497
498 /* Unlink a packet tag. */
499 void
m_tag_unlink(struct mbuf * m,struct m_tag * t)500 m_tag_unlink(struct mbuf *m, struct m_tag *t)
501 {
502 VERIFY(m->m_flags & M_PKTHDR);
503 VERIFY(t != NULL && t->m_tag_cookie == M_TAG_VALID_PATTERN);
504
505 SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
506 }
507
508 /* Unlink and free a packet tag. */
509 void
m_tag_delete(struct mbuf * m,struct m_tag * t)510 m_tag_delete(struct mbuf *m, struct m_tag *t)
511 {
512 m_tag_unlink(m, t);
513 m_tag_free(t);
514 }
515
516 /* Unlink and free a packet tag chain, starting from given tag. */
517 void
m_tag_delete_chain(struct mbuf * m,struct m_tag * t)518 m_tag_delete_chain(struct mbuf *m, struct m_tag *t)
519 {
520 struct m_tag *p, *q;
521
522 VERIFY(m->m_flags & M_PKTHDR);
523
524 if (t != NULL) {
525 p = t;
526 } else {
527 p = SLIST_FIRST(&m->m_pkthdr.tags);
528 }
529 if (p == NULL) {
530 return;
531 }
532
533 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
534 while ((q = SLIST_NEXT(p, m_tag_link)) != NULL) {
535 VERIFY(q->m_tag_cookie == M_TAG_VALID_PATTERN);
536 m_tag_delete(m, q);
537 }
538 m_tag_delete(m, p);
539 }
540
541 /* Find a tag, starting from a given position. */
542 struct m_tag *
m_tag_locate(struct mbuf * m,u_int32_t id,u_int16_t type,struct m_tag * t)543 m_tag_locate(struct mbuf *m, u_int32_t id, u_int16_t type, struct m_tag *t)
544 {
545 struct m_tag *p;
546
547 VERIFY(m->m_flags & M_PKTHDR);
548
549 if (t == NULL) {
550 p = SLIST_FIRST(&m->m_pkthdr.tags);
551 } else {
552 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
553 p = SLIST_NEXT(t, m_tag_link);
554 }
555 while (p != NULL) {
556 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
557 if (p->m_tag_id == id && p->m_tag_type == type) {
558 return p;
559 }
560 p = SLIST_NEXT(p, m_tag_link);
561 }
562 return NULL;
563 }
564
565 /* Copy a single tag. */
566 struct m_tag *
m_tag_copy(struct m_tag * t,int how)567 m_tag_copy(struct m_tag *t, int how)
568 {
569 struct m_tag *p;
570
571 VERIFY(t != NULL);
572
573 p = m_tag_alloc(t->m_tag_id, t->m_tag_type, t->m_tag_len, how);
574 if (p == NULL) {
575 return NULL;
576 }
577 bcopy(t + 1, p + 1, t->m_tag_len); /* Copy the data */
578 return p;
579 }
580
581 /*
582 * Copy two tag chains. The destination mbuf (to) loses any attached
583 * tags even if the operation fails. This should not be a problem, as
584 * m_tag_copy_chain() is typically called with a newly-allocated
585 * destination mbuf.
586 */
587 int
m_tag_copy_chain(struct mbuf * to,struct mbuf * from,int how)588 m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how)
589 {
590 struct m_tag *p, *t, *tprev = NULL;
591
592 VERIFY((to->m_flags & M_PKTHDR) && (from->m_flags & M_PKTHDR));
593
594 m_tag_delete_chain(to, NULL);
595 SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) {
596 VERIFY(p->m_tag_cookie == M_TAG_VALID_PATTERN);
597 t = m_tag_copy(p, how);
598 if (t == NULL) {
599 m_tag_delete_chain(to, NULL);
600 return 0;
601 }
602 if (tprev == NULL) {
603 SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link);
604 } else {
605 SLIST_INSERT_AFTER(tprev, t, m_tag_link);
606 tprev = t;
607 }
608 }
609 return 1;
610 }
611
612 /* Initialize dynamic and static tags on an mbuf. */
613 void
m_tag_init(struct mbuf * m,int all)614 m_tag_init(struct mbuf *m, int all)
615 {
616 VERIFY(m->m_flags & M_PKTHDR);
617
618 SLIST_INIT(&m->m_pkthdr.tags);
619 /*
620 * If the caller wants to preserve static mbuf tags
621 * (e.g. m_dup_pkthdr), don't zero them out.
622 */
623 if (all) {
624 bzero(&m->m_pkthdr.builtin_mtag._net_mtag,
625 sizeof(m->m_pkthdr.builtin_mtag._net_mtag));
626 }
627 }
628
629 /* Get first tag in chain. */
630 struct m_tag *
m_tag_first(struct mbuf * m)631 m_tag_first(struct mbuf *m)
632 {
633 VERIFY(m->m_flags & M_PKTHDR);
634
635 return SLIST_FIRST(&m->m_pkthdr.tags);
636 }
637
638 /* Get next tag in chain. */
639 struct m_tag *
m_tag_next(struct mbuf * m,struct m_tag * t)640 m_tag_next(struct mbuf *m, struct m_tag *t)
641 {
642 #pragma unused(m)
643 VERIFY(t != NULL);
644 VERIFY(t->m_tag_cookie == M_TAG_VALID_PATTERN);
645
646 return SLIST_NEXT(t, m_tag_link);
647 }
648
649 int
m_set_traffic_class(struct mbuf * m,mbuf_traffic_class_t tc)650 m_set_traffic_class(struct mbuf *m, mbuf_traffic_class_t tc)
651 {
652 u_int32_t val = MBUF_TC2SCVAL(tc); /* just the val portion */
653
654 return m_set_service_class(m, m_service_class_from_val(val));
655 }
656
657 mbuf_traffic_class_t
m_get_traffic_class(struct mbuf * m)658 m_get_traffic_class(struct mbuf *m)
659 {
660 return MBUF_SC2TC(m_get_service_class(m));
661 }
662
663 int
m_set_service_class(struct mbuf * m,mbuf_svc_class_t sc)664 m_set_service_class(struct mbuf *m, mbuf_svc_class_t sc)
665 {
666 int error = 0;
667
668 VERIFY(m->m_flags & M_PKTHDR);
669
670 if (MBUF_VALID_SC(sc)) {
671 m->m_pkthdr.pkt_svc = sc;
672 } else {
673 error = EINVAL;
674 }
675
676 return error;
677 }
678
679 mbuf_svc_class_t
m_get_service_class(struct mbuf * m)680 m_get_service_class(struct mbuf *m)
681 {
682 mbuf_svc_class_t sc;
683
684 VERIFY(m->m_flags & M_PKTHDR);
685
686 if (MBUF_VALID_SC(m->m_pkthdr.pkt_svc)) {
687 sc = m->m_pkthdr.pkt_svc;
688 } else {
689 sc = MBUF_SC_BE;
690 }
691
692 return sc;
693 }
694
695 mbuf_svc_class_t
m_service_class_from_idx(u_int32_t i)696 m_service_class_from_idx(u_int32_t i)
697 {
698 mbuf_svc_class_t sc = MBUF_SC_BE;
699
700 switch (i) {
701 case SCIDX_BK_SYS:
702 return MBUF_SC_BK_SYS;
703
704 case SCIDX_BK:
705 return MBUF_SC_BK;
706
707 case SCIDX_BE:
708 return MBUF_SC_BE;
709
710 case SCIDX_RD:
711 return MBUF_SC_RD;
712
713 case SCIDX_OAM:
714 return MBUF_SC_OAM;
715
716 case SCIDX_AV:
717 return MBUF_SC_AV;
718
719 case SCIDX_RV:
720 return MBUF_SC_RV;
721
722 case SCIDX_VI:
723 return MBUF_SC_VI;
724
725 case SCIDX_VO:
726 return MBUF_SC_VO;
727
728 case SCIDX_CTL:
729 return MBUF_SC_CTL;
730
731 default:
732 break;
733 }
734
735 VERIFY(0);
736 /* NOTREACHED */
737 return sc;
738 }
739
740 mbuf_svc_class_t
m_service_class_from_val(u_int32_t v)741 m_service_class_from_val(u_int32_t v)
742 {
743 mbuf_svc_class_t sc = MBUF_SC_BE;
744
745 switch (v) {
746 case SCVAL_BK_SYS:
747 return MBUF_SC_BK_SYS;
748
749 case SCVAL_BK:
750 return MBUF_SC_BK;
751
752 case SCVAL_BE:
753 return MBUF_SC_BE;
754
755 case SCVAL_RD:
756 return MBUF_SC_RD;
757
758 case SCVAL_OAM:
759 return MBUF_SC_OAM;
760
761 case SCVAL_AV:
762 return MBUF_SC_AV;
763
764 case SCVAL_RV:
765 return MBUF_SC_RV;
766
767 case SCVAL_VI:
768 return MBUF_SC_VI;
769
770 case SCVAL_VO:
771 return MBUF_SC_VO;
772
773 case SCVAL_CTL:
774 return MBUF_SC_CTL;
775
776 default:
777 break;
778 }
779
780 VERIFY(0);
781 /* NOTREACHED */
782 return sc;
783 }
784
785 uint16_t
m_adj_sum16(struct mbuf * m,uint32_t start,uint32_t dataoff,uint32_t datalen,uint32_t sum)786 m_adj_sum16(struct mbuf *m, uint32_t start, uint32_t dataoff,
787 uint32_t datalen, uint32_t sum)
788 {
789 uint32_t total_sub = 0; /* total to subtract */
790 uint32_t mlen = m_pktlen(m); /* frame length */
791 uint32_t bytes = (dataoff + datalen); /* bytes covered by sum */
792 int len;
793
794 ASSERT(bytes <= mlen);
795
796 /*
797 * Take care of excluding (len > 0) or including (len < 0)
798 * extraneous octets at the beginning of the packet, taking
799 * into account the start offset.
800 */
801 len = (dataoff - start);
802 if (len > 0) {
803 total_sub = m_sum16(m, start, len);
804 } else if (len < 0) {
805 sum += m_sum16(m, dataoff, -len);
806 }
807
808 /*
809 * Take care of excluding any postpended extraneous octets.
810 */
811 len = (mlen - bytes);
812 if (len > 0) {
813 struct mbuf *m0 = m;
814 uint32_t extra = m_sum16(m, bytes, len);
815 uint32_t off = bytes, off0 = off;
816
817 while (off > 0) {
818 if (__improbable(m == NULL)) {
819 panic("%s: invalid mbuf chain %p [off %u, "
820 "len %u]", __func__, m0, off0, len);
821 /* NOTREACHED */
822 }
823 if (off < m->m_len) {
824 break;
825 }
826 off -= m->m_len;
827 m = m->m_next;
828 }
829
830 /* if we started on odd-alignment, swap the value */
831 if ((uintptr_t)(mtod(m, uint8_t *) + off) & 1) {
832 total_sub += ((extra << 8) & 0xffff) | (extra >> 8);
833 } else {
834 total_sub += extra;
835 }
836
837 total_sub = (total_sub >> 16) + (total_sub & 0xffff);
838 }
839
840 /*
841 * 1's complement subtract any extraneous octets.
842 */
843 if (total_sub != 0) {
844 if (total_sub >= sum) {
845 sum = ~(total_sub - sum) & 0xffff;
846 } else {
847 sum -= total_sub;
848 }
849 }
850
851 /* fold 32-bit to 16-bit */
852 sum = (sum >> 16) + (sum & 0xffff); /* 17-bit */
853 sum = (sum >> 16) + (sum & 0xffff); /* 16-bit + carry */
854 sum = (sum >> 16) + (sum & 0xffff); /* final carry */
855
856 return sum & 0xffff;
857 }
858
859 uint16_t
m_sum16(struct mbuf * m,uint32_t off,uint32_t len)860 m_sum16(struct mbuf *m, uint32_t off, uint32_t len)
861 {
862 int mlen;
863
864 /*
865 * Sanity check
866 *
867 * Use m_length2() instead of m_length(), as we cannot rely on
868 * the caller setting m_pkthdr.len correctly, if the mbuf is
869 * a M_PKTHDR one.
870 */
871 if ((mlen = m_length2(m, NULL)) < (off + len)) {
872 panic("%s: mbuf %p len (%d) < off+len (%d+%d)", __func__,
873 m, mlen, off, len);
874 /* NOTREACHED */
875 }
876
877 return (uint16_t)os_cpu_in_cksum_mbuf(m, len, off, 0);
878 }
879