1 /* 2 * Copyright (c) 1999-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* Copyright (c) 1998, 1999 Apple Computer, Inc. All Rights Reserved */ 29 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ 30 /* 31 * Mach Operating System 32 * Copyright (c) 1987 Carnegie-Mellon University 33 * All rights reserved. The CMU software License Agreement specifies 34 * the terms and conditions for use and redistribution. 35 */ 36 /* 37 * Copyright (c) 1994 NeXT Computer, Inc. All rights reserved. 38 * 39 * Copyright (c) 1982, 1986, 1988 Regents of the University of California. 40 * All rights reserved. 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)mbuf.h 8.3 (Berkeley) 1/21/94 71 */ 72 /* 73 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce 74 * support for mandatory and extensible security protections. This notice 75 * is included in support of clause 2.2 (b) of the Apple Public License, 76 * Version 2.0. 77 */ 78 79 #ifndef _SYS_MBUF_H_ 80 #define _SYS_MBUF_H_ 81 82 #include <sys/appleapiopts.h> 83 #include <sys/cdefs.h> 84 #include <sys/_types/_u_int32_t.h> /* u_int32_t */ 85 #include <sys/_types/_u_int64_t.h> /* u_int64_t */ 86 #include <sys/_types/_u_short.h> /* u_short */ 87 88 #ifdef KERNEL 89 #include <sys/kpi_mbuf.h> 90 #endif 91 92 #ifdef XNU_KERNEL_PRIVATE 93 #include <sys/lock.h> 94 #include <sys/queue.h> 95 #include <machine/endian.h> 96 /* 97 * Mbufs are of a single size, MSIZE (machine/param.h), which 98 * includes overhead. An mbuf may add a single "mbuf cluster" of size 99 * MCLBYTES/MBIGCLBYTES/M16KCLBYTES (also in machine/param.h), which has 100 * no additional overhead and is used instead of the internal data area; 101 * this is done when at least MINCLSIZE of data must be stored. 102 */ 103 104 /* 105 * The following _MLEN and _MHLEN macros are private to xnu. Private code 106 * that are outside of xnu must use the mbuf_get_{mlen,mhlen} routines since 107 * the sizes of the structures are dependent upon specific xnu configs. 108 */ 109 #define _MLEN (MSIZE - sizeof(struct m_hdr)) /* normal data len */ 110 #define _MHLEN (_MLEN - sizeof(struct pkthdr)) /* data len w/pkthdr */ 111 112 #define NMBPGSHIFT (PAGE_SHIFT - MSIZESHIFT) 113 #define NMBPG (1 << NMBPGSHIFT) /* # of mbufs per page */ 114 115 #define NCLPGSHIFT (PAGE_SHIFT - MCLSHIFT) 116 #define NCLPG (1 << NCLPGSHIFT) /* # of cl per page */ 117 118 #define NBCLPGSHIFT (PAGE_SHIFT - MBIGCLSHIFT) 119 #define NBCLPG (1 << NBCLPGSHIFT) /* # of big cl per page */ 120 121 #define NMBPCLSHIFT (MCLSHIFT - MSIZESHIFT) 122 #define NMBPCL (1 << NMBPCLSHIFT) /* # of mbufs per cl */ 123 124 #define NCLPJCLSHIFT (M16KCLSHIFT - MCLSHIFT) 125 #define NCLPJCL (1 << NCLPJCLSHIFT) /* # of cl per jumbo cl */ 126 127 #define NCLPBGSHIFT (MBIGCLSHIFT - MCLSHIFT) 128 #define NCLPBG (1 << NCLPBGSHIFT) /* # of cl per big cl */ 129 130 #define NMBPBGSHIFT (MBIGCLSHIFT - MSIZESHIFT) 131 #define NMBPBG (1 << NMBPBGSHIFT) /* # of mbufs per big cl */ 132 133 /* 134 * Macros for type conversion 135 * mtod(m,t) - convert mbuf pointer to data pointer of correct type 136 * mtodo(m, o) -- Same as above but with offset 'o' into data. 137 * dtom(x) - convert data pointer within mbuf to mbuf pointer (XXX) 138 */ 139 #define mtod(m, t) ((t)m_mtod(m)) 140 #define mtodo(m, o) ((void *)(mtod(m, uint8_t *) + (o))) 141 #define dtom(x) m_dtom(x) 142 143 /* header at beginning of each mbuf: */ 144 struct m_hdr { 145 struct mbuf *mh_next; /* next buffer in chain */ 146 struct mbuf *mh_nextpkt; /* next chain in queue/record */ 147 caddr_t mh_data; /* location of data */ 148 int32_t mh_len; /* amount of data in this mbuf */ 149 u_int16_t mh_type; /* type of data in this mbuf */ 150 u_int16_t mh_flags; /* flags; see below */ 151 #if __arm__ && (__BIGGEST_ALIGNMENT__ > 4) 152 /* This is needed because of how _MLEN is defined and used. Ideally, _MLEN 153 * should be defined using the offsetof(struct mbuf, M_dat), since there is 154 * no guarantee that mbuf.M_dat will start where mbuf.m_hdr ends. The compiler 155 * may (and does in the armv7k case) insert padding between m_hdr and M_dat in 156 * mbuf. We cannot easily use offsetof, however, since _MLEN is referenced 157 * in the definition of mbuf. 158 */ 159 } __attribute__((aligned(8))); 160 #else 161 }; 162 #endif 163 164 /* 165 * Packet tag structure (see below for details). 166 */ 167 struct m_tag { 168 uint64_t m_tag_cookie; /* Error checking */ 169 #ifndef __LP64__ 170 uint32_t pad; /* For structure alignment */ 171 #endif /* !__LP64__ */ 172 SLIST_ENTRY(m_tag) m_tag_link; /* List of packet tags */ 173 uint16_t m_tag_type; /* Module specific type */ 174 uint16_t m_tag_len; /* Length of data */ 175 uint32_t m_tag_id; /* Module ID */ 176 }; 177 178 #define M_TAG_ALIGN(len) \ 179 (P2ROUNDUP(len, sizeof (u_int64_t)) + sizeof (struct m_tag)) 180 181 #define M_TAG_VALID_PATTERN 0xfeedfacefeedfaceULL 182 #define M_TAG_FREE_PATTERN 0xdeadbeefdeadbeefULL 183 184 /* 185 * Packet tag header structure (at the top of mbuf). Pointers are 186 * 32-bit in ILP32; m_tag needs 64-bit alignment, hence padded. 187 */ 188 struct m_taghdr { 189 #ifndef __LP64__ 190 u_int32_t pad; /* For structure alignment */ 191 #endif /* !__LP64__ */ 192 u_int64_t refcnt; /* Number of tags in this mbuf */ 193 }; 194 195 /* 196 * Driver auxiliary metadata tag (KERNEL_TAG_TYPE_DRVAUX). 197 */ 198 struct m_drvaux_tag { 199 u_int32_t da_family; /* IFNET_FAMILY values */ 200 u_int32_t da_subfamily; /* IFNET_SUBFAMILY values */ 201 u_int32_t da_reserved; /* for future */ 202 u_int32_t da_length; /* length of following data */ 203 }; 204 205 /* Values for pftag_flags (16-bit wide) */ 206 #define PF_TAG_GENERATED 0x1 /* pkt generated by PF */ 207 #define PF_TAG_FRAGCACHE 0x2 208 #define PF_TAG_TRANSLATE_LOCALHOST 0x4 209 #if PF_ECN 210 #define PF_TAG_HDR_INET 0x8 /* hdr points to IPv4 */ 211 #define PF_TAG_HDR_INET6 0x10 /* hdr points to IPv6 */ 212 #endif /* PF_ECN */ 213 #define PF_TAG_REASSEMBLED 0x20 /* pkt reassembled by PF */ 214 #define PF_TAG_REFRAGMENTED 0x40 /* pkt refragmented by PF */ 215 /* 216 * PF mbuf tag 217 */ 218 struct pf_mtag { 219 u_int16_t pftag_flags; /* PF_TAG flags */ 220 u_int16_t pftag_rtableid; /* alternate routing table id */ 221 u_int16_t pftag_tag; 222 u_int16_t pftag_routed; 223 #if PF_ECN 224 void *pftag_hdr; /* saved hdr pos in mbuf, for ECN */ 225 #endif /* PF_ECN */ 226 }; 227 228 /* System reserved PF tags */ 229 #define PF_TAG_ID_SYSTEM_SERVICE 0xff00 230 #define PF_TAG_ID_STACK_DROP 0xff01 231 232 /* 233 * PF fragment tag 234 */ 235 struct pf_fragment_tag { 236 uint32_t ft_id; /* fragment id */ 237 uint16_t ft_hdrlen; /* header length of reassembled pkt */ 238 uint16_t ft_unfragpartlen; /* length of the per-fragment headers */ 239 uint16_t ft_extoff; /* last extension header offset or 0 */ 240 uint16_t ft_maxlen; /* maximum fragment payload length */ 241 }; 242 243 /* 244 * TCP mbuf tag 245 */ 246 struct tcp_pktinfo { 247 union { 248 struct { 249 uint32_t segsz; /* segment size (actual MSS) */ 250 uint32_t start_seq; /* start seq of this packet */ 251 pid_t pid; 252 pid_t e_pid; 253 } __tx; 254 struct { 255 uint8_t seg_cnt; /* # of coalesced TCP pkts */ 256 } __rx; 257 } __offload; 258 #define tso_segsz proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.segsz 259 #define tx_start_seq proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.start_seq 260 #define tx_tcp_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.pid 261 #define tx_tcp_e_pid proto_mtag.__pr_u.tcp.tm_tcp.__offload.__tx.e_pid 262 #define seg_cnt proto_mtag.__pr_u.tcp.tm_tcp.__offload.__rx.seg_cnt 263 }; 264 265 /* 266 * MPTCP mbuf tag 267 */ 268 struct mptcp_pktinfo { 269 uint64_t mtpi_dsn; /* MPTCP Data Sequence Number */ 270 uint32_t mtpi_rel_seq; /* Relative Seq Number */ 271 uint16_t mtpi_length; /* Length of mapping */ 272 uint16_t mtpi_csum; 273 #define mp_dsn proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_dsn 274 #define mp_rseq proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_rel_seq 275 #define mp_rlen proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_length 276 #define mp_csum proto_mtag.__pr_u.tcp.tm_mptcp.mtpi_csum 277 }; 278 279 /* 280 * TCP specific mbuf tag. Note that the current implementation uses 281 * MPTCP metadata strictly between MPTCP and the TCP subflow layers, 282 * hence tm_tcp and tm_mptcp are mutually exclusive. This also means 283 * that TCP messages functionality is currently incompatible with MPTCP. 284 */ 285 struct tcp_mtag { 286 union { 287 struct tcp_pktinfo tm_tcp; /* TCP and below */ 288 struct mptcp_pktinfo tm_mptcp; /* MPTCP-TCP only */ 289 }; 290 }; 291 292 struct udp_mtag { 293 pid_t _pid; 294 pid_t _e_pid; 295 #define tx_udp_pid proto_mtag.__pr_u.udp._pid 296 #define tx_udp_e_pid proto_mtag.__pr_u.udp._e_pid 297 }; 298 299 struct rawip_mtag { 300 pid_t _pid; 301 pid_t _e_pid; 302 #define tx_rawip_pid proto_mtag.__pr_u.rawip._pid 303 #define tx_rawip_e_pid proto_mtag.__pr_u.rawip._e_pid 304 }; 305 306 struct driver_mtag_ { 307 uintptr_t _drv_tx_compl_arg; 308 uintptr_t _drv_tx_compl_data; 309 kern_return_t _drv_tx_status; 310 uint16_t _drv_flowid; 311 #define drv_tx_compl_arg builtin_mtag._drv_mtag._drv_tx_compl_arg 312 #define drv_tx_compl_data builtin_mtag._drv_mtag._drv_tx_compl_data 313 #define drv_tx_status builtin_mtag._drv_mtag._drv_tx_status 314 #define drv_flowid builtin_mtag._drv_mtag._drv_flowid 315 }; 316 317 /* 318 * Protocol specific mbuf tag (at most one protocol metadata per mbuf). 319 * 320 * Care must be taken to ensure that they are mutually exclusive, e.g. 321 * IPsec policy ID implies no TCP segment offload (which is fine given 322 * that the former is used on the virtual ipsec interface that does 323 * not advertise the TSO capability.) 324 */ 325 struct proto_mtag_ { 326 union { 327 struct tcp_mtag tcp; /* TCP specific */ 328 struct udp_mtag udp; /* UDP specific */ 329 struct rawip_mtag rawip; /* raw IPv4/IPv6 specific */ 330 } __pr_u; 331 }; 332 333 /* 334 * NECP specific mbuf tag. 335 */ 336 struct necp_mtag_ { 337 u_int32_t necp_policy_id; 338 u_int32_t necp_skip_policy_id; 339 u_int32_t necp_route_rule_id; 340 u_int16_t necp_last_interface_index; 341 u_int16_t necp_app_id; 342 }; 343 344 union builtin_mtag { 345 struct { 346 struct proto_mtag_ _proto_mtag; /* built-in protocol-specific tag */ 347 struct pf_mtag _pf_mtag; /* built-in PF tag */ 348 struct necp_mtag_ _necp_mtag; /* built-in NECP tag */ 349 } _net_mtag; 350 struct driver_mtag_ _drv_mtag; 351 #define necp_mtag builtin_mtag._net_mtag._necp_mtag 352 #define proto_mtag builtin_mtag._net_mtag._proto_mtag 353 #define driver_mtag builtin_mtag._drv_mtag 354 }; 355 356 /* 357 * Record/packet header in first mbuf of chain; valid only if M_PKTHDR set. 358 */ 359 struct pkthdr { 360 struct ifnet *rcvif; /* rcv interface */ 361 /* variables for ip and tcp reassembly */ 362 void *pkt_hdr; /* pointer to packet header */ 363 int32_t len; /* total packet length */ 364 /* variables for hardware checksum */ 365 /* Note: csum_flags is used for hardware checksum and VLAN */ 366 u_int32_t csum_flags; /* flags regarding checksum */ 367 union { 368 struct { 369 u_int16_t val; /* checksum value */ 370 u_int16_t start; /* checksum start offset */ 371 } _csum_rx; 372 #define csum_rx_val _csum_rx.val 373 #define csum_rx_start _csum_rx.start 374 struct { 375 u_int16_t start; /* checksum start offset */ 376 u_int16_t stuff; /* checksum stuff offset */ 377 } _csum_tx; 378 #define csum_tx_start _csum_tx.start 379 #define csum_tx_stuff _csum_tx.stuff 380 /* 381 * Generic data field used by csum routines. 382 * It gets used differently in different contexts. 383 */ 384 u_int32_t csum_data; 385 }; 386 u_int16_t vlan_tag; /* VLAN tag, host byte order */ 387 /* 388 * Packet classifier info 389 * 390 * PKTF_FLOW_ID set means valid flow ID. A non-zero flow ID value 391 * means the packet has been classified by one of the flow sources. 392 * It is also a prerequisite for flow control advisory, which is 393 * enabled by additionally setting PKTF_FLOW_ADV. 394 * 395 * The protocol value is a best-effort representation of the payload. 396 * It is opportunistically updated and used only for optimization. 397 * It is not a substitute for parsing the protocol header(s); use it 398 * only as a hint. 399 * 400 * If PKTF_IFAINFO is set, pkt_ifainfo contains one or both of the 401 * indices of interfaces which own the source and/or destination 402 * addresses of the packet. For the local/loopback case (PKTF_LOOP), 403 * both should be valid, and thus allows for the receiving end to 404 * quickly determine the actual interfaces used by the the addresses; 405 * they may not necessarily be the same or refer to the loopback 406 * interface. Otherwise, in the non-local/loopback case, the indices 407 * are opportunistically set, and because of that only one may be set 408 * (0 means the index has not been determined.) In addition, the 409 * interface address flags are also recorded. This allows us to avoid 410 * storing the corresponding {in,in6}_ifaddr in an mbuf tag. Ideally 411 * this would be a superset of {ia,ia6}_flags, but the namespaces are 412 * overlapping at present, so we'll need a new set of values in future 413 * to achieve this. For now, we will just rely on the address family 414 * related code paths examining this mbuf to interpret the flags. 415 */ 416 u_int8_t pkt_proto; /* IPPROTO value */ 417 u_int8_t pkt_flowsrc; /* FLOWSRC values */ 418 u_int32_t pkt_flowid; /* flow ID */ 419 u_int32_t pkt_flags; /* PKTF flags (see below) */ 420 u_int32_t pkt_svc; /* MBUF_SVC value */ 421 422 u_int32_t pkt_compl_context; /* Packet completion context */ 423 424 union { 425 struct { 426 u_int16_t src; /* ifindex of src addr i/f */ 427 u_int16_t src_flags; /* src PKT_IFAIFF flags */ 428 u_int16_t dst; /* ifindex of dst addr i/f */ 429 u_int16_t dst_flags; /* dst PKT_IFAIFF flags */ 430 } _pkt_iaif; 431 #define src_ifindex _pkt_iaif.src 432 #define src_iff _pkt_iaif.src_flags 433 #define dst_ifindex _pkt_iaif.dst 434 #define dst_iff _pkt_iaif.dst_flags 435 u_int64_t pkt_ifainfo; /* data field used by ifainfo */ 436 struct { 437 u_int32_t if_data; /* bytes in interface queue */ 438 u_int32_t sndbuf_data; /* bytes in socket buffer */ 439 } _pkt_bsr; /* Buffer status report used by cellular interface */ 440 #define bufstatus_if _pkt_bsr.if_data 441 #define bufstatus_sndbuf _pkt_bsr.sndbuf_data 442 }; 443 u_int64_t pkt_timestamp; /* TX: enqueue time, RX: receive timestamp */ 444 445 /* 446 * Tags (external and built-in) 447 */ 448 SLIST_HEAD(packet_tags, m_tag) tags; /* list of external tags */ 449 union builtin_mtag builtin_mtag; 450 451 uint32_t comp_gencnt; 452 uint16_t pkt_ext_flags; 453 uint16_t pkt_crumbs; 454 /* 455 * Module private scratch space (32-bit aligned), currently 16-bytes 456 * large. Anything stored here is not guaranteed to survive across 457 * modules. The AQM layer (outbound) uses all 16-bytes for both 458 * packet scheduling and flow advisory information. 459 */ 460 struct { 461 union { 462 u_int8_t __mpriv8[16]; 463 u_int16_t __mpriv16[8]; 464 struct { 465 union { 466 u_int8_t __val8[4]; 467 u_int16_t __val16[2]; 468 u_int32_t __val32; 469 } __mpriv32_u; 470 } __mpriv32[4]; 471 u_int64_t __mpriv64[2]; 472 } __mpriv_u; 473 } pkt_mpriv __attribute__((aligned(4))); 474 #define pkt_mpriv_hash pkt_mpriv.__mpriv_u.__mpriv32[0].__mpriv32_u.__val32 475 #define pkt_mpriv_flags pkt_mpriv.__mpriv_u.__mpriv32[1].__mpriv32_u.__val32 476 #define pkt_mpriv_srcid pkt_mpriv.__mpriv_u.__mpriv32[2].__mpriv32_u.__val32 477 #define pkt_mpriv_fidx pkt_mpriv.__mpriv_u.__mpriv32[3].__mpriv32_u.__val32 478 479 u_int32_t redzone; /* red zone */ 480 u_int32_t pkt_compl_callbacks; /* Packet completion callbacks */ 481 }; 482 483 /* 484 * Flow data source type. A data source module is responsible for generating 485 * a unique flow ID and associating it to each data flow as pkt_flowid. 486 * This is required for flow control/advisory, as it allows the output queue 487 * to identify the data source object and inform that it can resume its 488 * transmission (in the event it was flow controlled.) 489 */ 490 #define FLOWSRC_INPCB 1 /* flow ID generated by INPCB */ 491 #define FLOWSRC_IFNET 2 /* flow ID generated by interface */ 492 #define FLOWSRC_PF 3 /* flow ID generated by PF */ 493 #define FLOWSRC_CHANNEL 4 /* flow ID generated by channel */ 494 495 /* 496 * FLOWSRC_MPKL_INPUT is not a true flow data source and is used for 497 * multi-layer packet logging. We're usurping the pkt_flowsrc field because 498 * the mbuf packet header ran out of space and pkt_flowsrc is normally 499 * used on output so we assume we can safely overwrite the normal semantic of 500 * the field. 501 * This value is meant to be used on incoming packet from a lower level protocol 502 * to pass information to some upper level protocol. When FLOWSRC_MPKL_INPUT 503 * is set, the following fields are used: 504 * - pkt_proto: the IP protocol ID of the lower level protocol 505 * - pkt_flowid: the identifier of the packet at the lower protocol. 506 * For example ESP would set pkt_proto to IPPROTO_ESP and pkt_flowid to the SPI. 507 */ 508 509 /* 510 * Packet flags. Unlike m_flags, all packet flags are copied along when 511 * copying m_pkthdr, i.e. no equivalent of M_COPYFLAGS here. These flags 512 * (and other classifier info) will be cleared during DLIL input. 513 * 514 * Some notes about M_LOOP and PKTF_LOOP: 515 * 516 * - M_LOOP flag is overloaded, and its use is discouraged. Historically, 517 * that flag was used by the KAME implementation for allowing certain 518 * certain exceptions to be made in the IP6_EXTHDR_CHECK() logic; this 519 * was originally meant to be set as the packet is looped back to the 520 * system, and in some circumstances temporarily set in ip6_output(). 521 * Over time, this flag was used by the pre-output routines to indicate 522 * to the DLIL frameout and output routines, that the packet may be 523 * looped back to the system under the right conditions. In addition, 524 * this is an mbuf flag rather than an mbuf packet header flag. 525 * 526 * - PKTF_LOOP is an mbuf packet header flag, which is set if and only 527 * if the packet was looped back to the system. This flag should be 528 * used instead for newer code. 529 */ 530 #define PKTF_FLOW_ID 0x1 /* pkt has valid flowid value */ 531 #define PKTF_FLOW_ADV 0x2 /* pkt triggers local flow advisory */ 532 #define PKTF_FLOW_LOCALSRC 0x4 /* pkt is locally originated */ 533 #define PKTF_FLOW_RAWSOCK 0x8 /* pkt locally generated by raw sock */ 534 #define PKTF_PRIO_PRIVILEGED 0x10 /* packet priority is privileged */ 535 #define PKTF_PROXY_DST 0x20 /* processed but not locally destined */ 536 #define PKTF_INET_RESOLVE 0x40 /* IPv4 resolver packet */ 537 #define PKTF_INET6_RESOLVE 0x80 /* IPv6 resolver packet */ 538 #define PKTF_RESOLVE_RTR 0x100 /* pkt is for resolving router */ 539 #define PKTF_SKIP_PKTAP 0x200 /* pkt has already passed through pktap */ 540 #define PKTF_WAKE_PKT 0x400 /* packet caused system to wake from sleep */ 541 #define PKTF_MPTCP 0x800 /* TCP with MPTCP metadata */ 542 #define PKTF_MPSO 0x1000 /* MPTCP socket meta data */ 543 #define PKTF_LOOP 0x2000 /* loopbacked packet */ 544 #define PKTF_IFAINFO 0x4000 /* pkt has valid interface addr info */ 545 #define PKTF_SO_BACKGROUND 0x8000 /* data is from background source */ 546 #define PKTF_FORWARDED 0x10000 /* pkt was forwarded from another i/f */ 547 #define PKTF_PRIV_GUARDED 0x20000 /* pkt_mpriv area guard enabled */ 548 #define PKTF_KEEPALIVE 0x40000 /* pkt is kernel-generated keepalive */ 549 #define PKTF_SO_REALTIME 0x80000 /* data is realtime traffic */ 550 #define PKTF_VALID_UNSENT_DATA 0x100000 /* unsent data is valid */ 551 #define PKTF_TCP_REXMT 0x200000 /* packet is TCP retransmission */ 552 #define PKTF_REASSEMBLED 0x400000 /* Packet was reassembled */ 553 #define PKTF_TX_COMPL_TS_REQ 0x800000 /* tx completion timestamp requested */ 554 #define PKTF_TS_VALID 0x1000000 /* pkt timestamp is valid */ 555 #define PKTF_DRIVER_MTAG 0x2000000 /* driver mbuf tags fields inited */ 556 #define PKTF_NEW_FLOW 0x4000000 /* Data from a new flow */ 557 #define PKTF_START_SEQ 0x8000000 /* valid start sequence */ 558 #define PKTF_LAST_PKT 0x10000000 /* last packet in the flow */ 559 #define PKTF_MPTCP_REINJ 0x20000000 /* Packet has been reinjected for MPTCP */ 560 #define PKTF_MPTCP_DFIN 0x40000000 /* Packet is a data-fin */ 561 #define PKTF_HBH_CHKED 0x80000000 /* HBH option is checked */ 562 563 #define PKTF_EXT_OUTPUT_SCOPE 0x1 /* outgoing packet has ipv6 address scope id */ 564 565 #define PKT_CRUMB_TS_COMP_REQ 0x0001 /* timestamp completion requested */ 566 #define PKT_CRUMB_TS_COMP_CB 0x0002 /* timestamp callback called */ 567 #define PKT_CRUMB_DLIL_OUTPUT 0x0004 /* dlil_output called */ 568 #define PKT_CRUMB_FLOW_TX 0x0008 /* dp_flow_tx_process called */ 569 #define PKT_CRUMB_FQ_ENQUEUE 0x0010 /* fq_enqueue called */ 570 #define PKT_CRUMB_FQ_DEQUEUE 0x0020 /* fq_dequeue called */ 571 #define PKT_CRUMB_SK_PKT_COPY 0x0040 /* copy from mbuf to skywalk packet */ 572 #define PKT_CRUMB_TCP_OUTPUT 0x0080 573 #define PKT_CRUMB_UDP_OUTPUT 0x0100 574 #define PKT_CRUMB_SOSEND 0x0200 575 #define PKT_CRUMB_DLIL_INPUT 0x0400 576 #define PKT_CRUMB_IP_INPUT 0x0800 577 #define PKT_CRUMB_TCP_INPUT 0x1000 578 #define PKT_CRUMB_UDP_INPUT 0x2000 579 580 /* flags related to flow control/advisory and identification */ 581 #define PKTF_FLOW_MASK \ 582 (PKTF_FLOW_ID | PKTF_FLOW_ADV | PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK) 583 584 /* 585 * Description of external storage mapped into mbuf, valid only if M_EXT set. 586 */ 587 typedef void (*m_ext_free_func_t)(caddr_t, u_int, caddr_t); 588 struct m_ext { 589 caddr_t ext_buf; /* start of buffer */ 590 m_ext_free_func_t ext_free; /* free routine if not the usual */ 591 u_int ext_size; /* size of buffer, for ext_free */ 592 caddr_t ext_arg; /* additional ext_free argument */ 593 struct ext_ref { 594 struct mbuf *paired; 595 u_int16_t minref; 596 u_int16_t refcnt; 597 u_int16_t prefcnt; 598 u_int16_t flags; 599 u_int32_t priv; 600 uintptr_t ext_token; 601 } *ext_refflags; 602 }; 603 604 /* define m_ext to a type since it gets redefined below */ 605 typedef struct m_ext _m_ext_t; 606 607 /* 608 * The mbuf object 609 */ 610 struct mbuf { 611 struct m_hdr m_hdr; 612 union { 613 struct { 614 struct pkthdr MH_pkthdr; /* M_PKTHDR set */ 615 union { 616 struct m_ext MH_ext; /* M_EXT set */ 617 char MH_databuf[_MHLEN]; 618 } MH_dat; 619 } MH; 620 char M_databuf[_MLEN]; /* !M_PKTHDR, !M_EXT */ 621 } M_dat; 622 }; 623 624 #define m_next m_hdr.mh_next 625 #define m_len m_hdr.mh_len 626 #define m_data m_hdr.mh_data 627 #define m_type m_hdr.mh_type 628 #define m_flags m_hdr.mh_flags 629 #define m_nextpkt m_hdr.mh_nextpkt 630 #define m_act m_nextpkt 631 #define m_pkthdr M_dat.MH.MH_pkthdr 632 #define m_ext M_dat.MH.MH_dat.MH_ext 633 #define m_pktdat M_dat.MH.MH_dat.MH_databuf 634 #define m_dat M_dat.M_databuf 635 #define m_pktlen(_m) ((_m)->m_pkthdr.len) 636 #define m_pftag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._pf_mtag) 637 #define m_necptag(_m) (&(_m)->m_pkthdr.builtin_mtag._net_mtag._necp_mtag) 638 639 /* mbuf flags (private) */ 640 #define M_EXT 0x0001 /* has associated external storage */ 641 #define M_PKTHDR 0x0002 /* start of record */ 642 #define M_EOR 0x0004 /* end of record */ 643 #define M_PROTO1 0x0008 /* protocol-specific */ 644 #define M_PROTO2 0x0010 /* protocol-specific */ 645 #define M_PROTO3 0x0020 /* protocol-specific */ 646 #define M_LOOP 0x0040 /* packet is looped back (also see PKTF_LOOP) */ 647 #define M_PROTO5 0x0080 /* protocol-specific */ 648 649 /* mbuf pkthdr flags, also in m_flags (private) */ 650 #define M_BCAST 0x0100 /* send/received as link-level broadcast */ 651 #define M_MCAST 0x0200 /* send/received as link-level multicast */ 652 #define M_FRAG 0x0400 /* packet is a fragment of a larger packet */ 653 #define M_FIRSTFRAG 0x0800 /* packet is first fragment */ 654 #define M_LASTFRAG 0x1000 /* packet is last fragment */ 655 #define M_PROMISC 0x2000 /* packet is promiscuous (shouldn't go to stack) */ 656 #define M_HASFCS 0x4000 /* packet has FCS */ 657 #define M_TAGHDR 0x8000 /* m_tag hdr structure at top of mbuf data */ 658 659 /* 660 * Flags to purge when crossing layers. 661 */ 662 #define M_PROTOFLAGS \ 663 (M_PROTO1|M_PROTO2|M_PROTO3|M_PROTO5) 664 665 /* flags copied when copying m_pkthdr */ 666 #define M_COPYFLAGS \ 667 (M_PKTHDR|M_EOR|M_PROTO1|M_PROTO2|M_PROTO3 | \ 668 M_LOOP|M_PROTO5|M_BCAST|M_MCAST|M_FRAG | \ 669 M_FIRSTFRAG|M_LASTFRAG|M_PROMISC|M_HASFCS) 670 671 /* flags indicating hw checksum support and sw checksum requirements */ 672 #define CSUM_IP 0x0001 /* will csum IP */ 673 #define CSUM_TCP 0x0002 /* will csum TCP */ 674 #define CSUM_UDP 0x0004 /* will csum UDP */ 675 #define CSUM_IP_FRAGS 0x0008 /* will csum IP fragments */ 676 #define CSUM_FRAGMENT 0x0010 /* will do IP fragmentation */ 677 #define CSUM_TCPIPV6 0x0020 /* will csum TCP for IPv6 */ 678 #define CSUM_UDPIPV6 0x0040 /* will csum UDP for IPv6 */ 679 #define CSUM_FRAGMENT_IPV6 0x0080 /* will do IPv6 fragmentation */ 680 681 #define CSUM_IP_CHECKED 0x0100 /* did csum IP */ 682 #define CSUM_IP_VALID 0x0200 /* ... the csum is valid */ 683 #define CSUM_DATA_VALID 0x0400 /* csum_data field is valid */ 684 #define CSUM_PSEUDO_HDR 0x0800 /* csum_data has pseudo hdr */ 685 #define CSUM_PARTIAL 0x1000 /* simple Sum16 computation */ 686 #define CSUM_ZERO_INVERT 0x2000 /* invert 0 to -0 (0xffff) */ 687 688 #define CSUM_DELAY_DATA (CSUM_TCP | CSUM_UDP) 689 #define CSUM_DELAY_IP (CSUM_IP) /* IPv4 only: no IPv6 IP cksum */ 690 #define CSUM_DELAY_IPV6_DATA (CSUM_TCPIPV6 | CSUM_UDPIPV6) 691 #define CSUM_DATA_IPV6_VALID CSUM_DATA_VALID /* csum_data field is valid */ 692 693 #define CSUM_TX_FLAGS \ 694 (CSUM_DELAY_IP | CSUM_DELAY_DATA | CSUM_DELAY_IPV6_DATA | \ 695 CSUM_DATA_VALID | CSUM_PARTIAL | CSUM_ZERO_INVERT) 696 697 #define CSUM_RX_FULL_FLAGS \ 698 (CSUM_IP_CHECKED | CSUM_IP_VALID | CSUM_PSEUDO_HDR | \ 699 CSUM_DATA_VALID) 700 701 #define CSUM_RX_FLAGS \ 702 (CSUM_RX_FULL_FLAGS | CSUM_PARTIAL) 703 704 705 706 /* 707 * Note: see also IF_HWASSIST_CSUM defined in <net/if_var.h> 708 */ 709 710 /* VLAN tag present */ 711 #define CSUM_VLAN_TAG_VALID 0x00010000 /* vlan_tag field is valid */ 712 713 /* checksum start adjustment has been done */ 714 #define CSUM_ADJUST_DONE 0x00020000 715 716 /* VLAN encapsulation present */ 717 #define CSUM_VLAN_ENCAP_PRESENT 0x00040000 /* mbuf has vlan encapsulation */ 718 719 /* TCP Segment Offloading requested on this mbuf */ 720 #define CSUM_TSO_IPV4 0x00100000 /* This mbuf needs to be segmented by the NIC */ 721 #define CSUM_TSO_IPV6 0x00200000 /* This mbuf needs to be segmented by the NIC */ 722 723 #define TSO_IPV4_OK(_ifp, _m) \ 724 (((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 725 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 726 727 #define TSO_IPV4_NOTOK(_ifp, _m) \ 728 (!((_ifp)->if_hwassist & IFNET_TSO_IPV4) && \ 729 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV4)) \ 730 731 #define TSO_IPV6_OK(_ifp, _m) \ 732 (((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 733 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 734 735 #define TSO_IPV6_NOTOK(_ifp, _m) \ 736 (!((_ifp)->if_hwassist & IFNET_TSO_IPV6) && \ 737 ((_m)->m_pkthdr.csum_flags & CSUM_TSO_IPV6)) \ 738 739 #endif /* XNU_KERNEL_PRIVATE */ 740 741 /* mbuf types */ 742 #define MT_FREE 0 /* should be on free list */ 743 #define MT_DATA 1 /* dynamic (data) allocation */ 744 #define MT_HEADER 2 /* packet header */ 745 #define MT_SOCKET 3 /* socket structure */ 746 #define MT_PCB 4 /* protocol control block */ 747 #define MT_RTABLE 5 /* routing tables */ 748 #define MT_HTABLE 6 /* IMP host tables */ 749 #define MT_ATABLE 7 /* address resolution tables */ 750 #define MT_SONAME 8 /* socket name */ 751 #define MT_SOOPTS 10 /* socket options */ 752 #define MT_FTABLE 11 /* fragment reassembly header */ 753 #define MT_RIGHTS 12 /* access rights */ 754 #define MT_IFADDR 13 /* interface address */ 755 #define MT_CONTROL 14 /* extra-data protocol message */ 756 #define MT_OOBDATA 15 /* expedited data */ 757 #define MT_TAG 16 /* volatile metadata associated to pkts */ 758 #define MT_MAX 32 /* enough? */ 759 760 #ifdef XNU_KERNEL_PRIVATE 761 /* 762 * mbuf allocation/deallocation macros: 763 * 764 * MGET(struct mbuf *m, int how, int type) 765 * allocates an mbuf and initializes it to contain internal data. 766 * 767 * MGETHDR(struct mbuf *m, int how, int type) 768 * allocates an mbuf and initializes it to contain a packet header 769 * and internal data. 770 */ 771 772 #if 1 773 #define MCHECK(m) m_mcheck(m) 774 #else 775 #define MCHECK(m) 776 #endif 777 778 #define MGET(m, how, type) ((m) = m_get((how), (type))) 779 780 #define MGETHDR(m, how, type) ((m) = m_gethdr((how), (type))) 781 782 /* 783 * Mbuf cluster macros. 784 * MCLALLOC(caddr_t p, int how) allocates an mbuf cluster. 785 * MCLGET adds such clusters to a normal mbuf; 786 * the flag M_EXT is set upon success. 787 * MCLFREE releases a reference to a cluster allocated by MCLALLOC, 788 * freeing the cluster if the reference count has reached 0. 789 * 790 * Normal mbuf clusters are normally treated as character arrays 791 * after allocation, but use the first word of the buffer as a free list 792 * pointer while on the free list. 793 */ 794 union mcluster { 795 union mcluster *mcl_next; 796 char mcl_buf[MCLBYTES]; 797 }; 798 799 #define MCLALLOC(p, how) ((p) = m_mclalloc(how)) 800 801 #define MCLFREE(p) m_mclfree(p) 802 803 #define MCLGET(m, how) ((m) = m_mclget(m, how)) 804 805 /* 806 * Mbuf big cluster 807 */ 808 union mbigcluster { 809 union mbigcluster *mbc_next; 810 char mbc_buf[MBIGCLBYTES]; 811 }; 812 813 /* 814 * Mbuf jumbo cluster 815 */ 816 union m16kcluster { 817 union m16kcluster *m16kcl_next; 818 char m16kcl_buf[M16KCLBYTES]; 819 }; 820 821 #define MCLHASREFERENCE(m) m_mclhasreference(m) 822 823 /* 824 * MFREE(struct mbuf *m, struct mbuf *n) 825 * Free a single mbuf and associated external storage. 826 * Place the successor, if any, in n. 827 */ 828 829 #define MFREE(m, n) ((n) = m_free(m)) 830 831 /* 832 * Copy mbuf pkthdr from from to to. 833 * from must have M_PKTHDR set, and to must be empty. 834 * aux pointer will be moved to `to'. 835 */ 836 #define M_COPY_PKTHDR(to, from) m_copy_pkthdr(to, from) 837 838 #define M_COPY_PFTAG(to, from) m_copy_pftag(to, from) 839 840 #define M_COPY_NECPTAG(to, from) m_copy_necptag(to, from) 841 842 #define M_COPY_CLASSIFIER(to, from) m_copy_classifier(to, from) 843 844 /* 845 * Evaluate TRUE if it's safe to write to the mbuf m's data region (this can 846 * be both the local data payload, or an external buffer area, depending on 847 * whether M_EXT is set). 848 */ 849 #define M_WRITABLE(m) (((m)->m_flags & M_EXT) == 0 || !MCLHASREFERENCE(m)) 850 851 /* 852 * These macros are mapped to the appropriate KPIs, so that private code 853 * can be simply recompiled in order to be forward-compatible with future 854 * changes toward the struture sizes. 855 */ 856 #ifdef XNU_KERNEL_PRIVATE 857 #define MLEN _MLEN 858 #define MHLEN _MHLEN 859 #define MINCLSIZE (MLEN + MHLEN) 860 #else 861 #define MLEN mbuf_get_mlen() /* normal mbuf data len */ 862 #define MHLEN mbuf_get_mhlen() /* data len in an mbuf w/pkthdr */ 863 #define MINCLSIZE mbuf_get_minclsize() /* cluster usage threshold */ 864 #endif 865 /* 866 * Return the address of the start of the buffer associated with an mbuf, 867 * handling external storage, packet-header mbufs, and regular data mbufs. 868 */ 869 #define M_START(m) \ 870 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_buf : \ 871 ((m)->m_flags & M_PKTHDR) ? &(m)->m_pktdat[0] : \ 872 &(m)->m_dat[0]) 873 874 /* 875 * Return the size of the buffer associated with an mbuf, handling external 876 * storage, packet-header mbufs, and regular data mbufs. 877 */ 878 #define M_SIZE(m) \ 879 (((m)->m_flags & M_EXT) ? (m)->m_ext.ext_size : \ 880 ((m)->m_flags & M_PKTHDR) ? MHLEN : \ 881 MLEN) 882 883 #define M_ALIGN(m, len) m_align(m, len) 884 #define MH_ALIGN(m, len) m_align(m, len) 885 #define MEXT_ALIGN(m, len) m_align(m, len) 886 887 /* 888 * Compute the amount of space available before the current start of data in 889 * an mbuf. 890 * 891 * The M_WRITABLE() is a temporary, conservative safety measure: the burden 892 * of checking writability of the mbuf data area rests solely with the caller. 893 */ 894 #define M_LEADINGSPACE(m) \ 895 (M_WRITABLE(m) ? ((m)->m_data - M_START(m)) : 0) 896 897 /* 898 * Compute the amount of space available after the end of data in an mbuf. 899 * 900 * The M_WRITABLE() is a temporary, conservative safety measure: the burden 901 * of checking writability of the mbuf data area rests solely with the caller. 902 */ 903 #define M_TRAILINGSPACE(m) \ 904 (M_WRITABLE(m) ? \ 905 ((M_START(m) + M_SIZE(m)) - ((m)->m_data + (m)->m_len)) : 0) 906 907 /* 908 * Arrange to prepend space of size plen to mbuf m. 909 * If a new mbuf must be allocated, how specifies whether to wait. 910 * If how is M_DONTWAIT and allocation fails, the original mbuf chain 911 * is freed and m is set to NULL. 912 */ 913 #define M_PREPEND(m, plen, how, align) \ 914 ((m) = m_prepend_2((m), (plen), (how), (align))) 915 916 /* change mbuf to new type */ 917 #define MCHTYPE(m, t) m_mchtype(m, t) 918 919 /* compatiblity with 4.3 */ 920 #define m_copy(m, o, l) m_copym((m), (o), (l), M_DONTWAIT) 921 922 #define MBSHIFT 20 /* 1MB */ 923 #define MBSIZE (1 << MBSHIFT) 924 #define GBSHIFT 30 /* 1GB */ 925 #define GBSIZE (1 << GBSHIFT) 926 927 /* 928 * M_STRUCT_GET ensures that intermediate protocol header (from "off" to 929 * "off+len") is located in single mbuf, on contiguous memory region. 930 * The pointer to the region will be returned to pointer variable "val", 931 * with type "typ". 932 * 933 * M_STRUCT_GET0 does the same, except that it aligns the structure at 934 * very top of mbuf. GET0 is likely to make memory copy than GET. 935 */ 936 #define M_STRUCT_GET(val, typ, m, off, len) \ 937 do { \ 938 struct mbuf *t; \ 939 int tmp; \ 940 \ 941 if ((m)->m_len >= (off) + (len)) { \ 942 (val) = (typ)(mtod((m), caddr_t) + (off)); \ 943 } else { \ 944 t = m_pulldown((m), (off), (len), &tmp); \ 945 if (t != NULL) { \ 946 if (t->m_len < tmp + (len)) \ 947 panic("m_pulldown malfunction"); \ 948 (val) = (typ)(mtod(t, caddr_t) + tmp); \ 949 } else { \ 950 (val) = (typ)NULL; \ 951 (m) = NULL; \ 952 } \ 953 } \ 954 } while (0) 955 956 #define M_STRUCT_GET0(val, typ, m, off, len) \ 957 do { \ 958 struct mbuf *t; \ 959 \ 960 if ((off) == 0 && ((m)->m_len >= (len))) { \ 961 (val) = (typ)(void *)mtod(m, caddr_t); \ 962 } else { \ 963 t = m_pulldown((m), (off), (len), NULL); \ 964 if (t != NULL) { \ 965 if (t->m_len < (len)) \ 966 panic("m_pulldown malfunction"); \ 967 (val) = (typ)(void *)mtod(t, caddr_t); \ 968 } else { \ 969 (val) = (typ)NULL; \ 970 (m) = NULL; \ 971 } \ 972 } \ 973 } while (0) 974 975 #define MBUF_INPUT_CHECK(m, rcvif) \ 976 do { \ 977 if (!(m->m_flags & MBUF_PKTHDR) || \ 978 m->m_len < 0 || \ 979 m->m_len > ((njcl > 0) ? njclbytes : MBIGCLBYTES) || \ 980 m->m_type == MT_FREE || \ 981 ((m->m_flags & M_EXT) != 0 && m->m_ext.ext_buf == NULL)) { \ 982 panic_plain("Failed mbuf validity check: mbuf %p len %d " \ 983 "type %d flags 0x%x data %p rcvif %s ifflags 0x%x", \ 984 m, m->m_len, m->m_type, m->m_flags, \ 985 ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : m->m_data), \ 986 if_name(rcvif), \ 987 (rcvif->if_flags & 0xffff)); \ 988 } \ 989 } while (0) 990 991 /* 992 * Simple mbuf queueing system 993 * 994 * This is basically a SIMPLEQ adapted to mbuf use (i.e. using 995 * m_nextpkt instead of field.sqe_next). 996 * 997 * m_next is ignored, so queueing chains of mbufs is possible 998 */ 999 #define MBUFQ_HEAD(name) \ 1000 struct name { \ 1001 struct mbuf *mq_first; /* first packet */ \ 1002 struct mbuf **mq_last; /* addr of last next packet */ \ 1003 } 1004 1005 #define MBUFQ_INIT(q) do { \ 1006 MBUFQ_FIRST(q) = NULL; \ 1007 (q)->mq_last = &MBUFQ_FIRST(q); \ 1008 } while (0) 1009 1010 #define MBUFQ_PREPEND(q, m) do { \ 1011 if ((MBUFQ_NEXT(m) = MBUFQ_FIRST(q)) == NULL) \ 1012 (q)->mq_last = &MBUFQ_NEXT(m); \ 1013 MBUFQ_FIRST(q) = (m); \ 1014 } while (0) 1015 1016 #define MBUFQ_ENQUEUE(q, m) do { \ 1017 MBUFQ_NEXT(m) = NULL; \ 1018 *(q)->mq_last = (m); \ 1019 (q)->mq_last = &MBUFQ_NEXT(m); \ 1020 } while (0) 1021 1022 #define MBUFQ_ENQUEUE_MULTI(q, m, n) do { \ 1023 MBUFQ_NEXT(n) = NULL; \ 1024 *(q)->mq_last = (m); \ 1025 (q)->mq_last = &MBUFQ_NEXT(n); \ 1026 } while (0) 1027 1028 #define MBUFQ_DEQUEUE(q, m) do { \ 1029 if (((m) = MBUFQ_FIRST(q)) != NULL) { \ 1030 if ((MBUFQ_FIRST(q) = MBUFQ_NEXT(m)) == NULL) \ 1031 (q)->mq_last = &MBUFQ_FIRST(q); \ 1032 else \ 1033 MBUFQ_NEXT(m) = NULL; \ 1034 } \ 1035 } while (0) 1036 1037 #define MBUFQ_REMOVE(q, m) do { \ 1038 if (MBUFQ_FIRST(q) == (m)) { \ 1039 MBUFQ_DEQUEUE(q, m); \ 1040 } else { \ 1041 struct mbuf *_m = MBUFQ_FIRST(q); \ 1042 while (MBUFQ_NEXT(_m) != (m)) \ 1043 _m = MBUFQ_NEXT(_m); \ 1044 if ((MBUFQ_NEXT(_m) = \ 1045 MBUFQ_NEXT(MBUFQ_NEXT(_m))) == NULL) \ 1046 (q)->mq_last = &MBUFQ_NEXT(_m); \ 1047 } \ 1048 } while (0) 1049 1050 #define MBUFQ_DRAIN(q) do { \ 1051 struct mbuf *__m0; \ 1052 while ((__m0 = MBUFQ_FIRST(q)) != NULL) { \ 1053 MBUFQ_FIRST(q) = MBUFQ_NEXT(__m0); \ 1054 MBUFQ_NEXT(__m0) = NULL; \ 1055 m_freem(__m0); \ 1056 } \ 1057 (q)->mq_last = &MBUFQ_FIRST(q); \ 1058 } while (0) 1059 1060 #define MBUFQ_FOREACH(m, q) \ 1061 for ((m) = MBUFQ_FIRST(q); \ 1062 (m); \ 1063 (m) = MBUFQ_NEXT(m)) 1064 1065 #define MBUFQ_FOREACH_SAFE(m, q, tvar) \ 1066 for ((m) = MBUFQ_FIRST(q); \ 1067 (m) && ((tvar) = MBUFQ_NEXT(m), 1); \ 1068 (m) = (tvar)) 1069 1070 #define MBUFQ_EMPTY(q) ((q)->mq_first == NULL) 1071 #define MBUFQ_FIRST(q) ((q)->mq_first) 1072 #define MBUFQ_NEXT(m) ((m)->m_nextpkt) 1073 /* 1074 * mq_last is initialized to point to mq_first, so check if they're 1075 * equal and return NULL when the list is empty. Otherwise, we need 1076 * to subtract the offset of MBUQ_NEXT (i.e. m_nextpkt field) to get 1077 * to the base mbuf address to return to caller. 1078 */ 1079 #define MBUFQ_LAST(head) \ 1080 (((head)->mq_last == &MBUFQ_FIRST(head)) ? NULL : \ 1081 ((struct mbuf *)(void *)((char *)(head)->mq_last - \ 1082 __builtin_offsetof(struct mbuf, m_nextpkt)))) 1083 1084 #if (DEBUG || DEVELOPMENT) 1085 #define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f) do { \ 1086 struct mbuf * _saved = (_t)->m_nextpkt; \ 1087 struct mbuf * _m; \ 1088 for (_m = (_h); _m != NULL; _m = MBUFQ_NEXT(_m)) { \ 1089 m_add_crumb((_m), (_f)); \ 1090 } \ 1091 (_t)->m_nextpkt = _saved; \ 1092 } while (0) 1093 1094 #define MBUFQ_ADD_CRUMB(_q, _m, _f) do { \ 1095 m_add_crumb((_m), (_f)); \ 1096 } while (0) 1097 #else 1098 #define MBUFQ_ADD_CRUMB_MULTI(_q, _h, _t, _f) 1099 #define MBUFQ_ADD_CRUMB(_q, _m, _f) 1100 #endif /* (DEBUG || DEVELOPMENT) */ 1101 1102 #define max_linkhdr (int)P2ROUNDUP(_max_linkhdr, sizeof (uint32_t)) 1103 #define max_protohdr (int)P2ROUNDUP(_max_protohdr, sizeof (uint32_t)) 1104 #endif /* XNU_KERNEL_PRIVATE */ 1105 1106 /* 1107 * Mbuf statistics (legacy). 1108 */ 1109 struct mbstat { 1110 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 1111 u_int32_t m_clusters; /* clusters obtained from page pool */ 1112 u_int32_t m_spare; /* spare field */ 1113 u_int32_t m_clfree; /* free clusters */ 1114 u_int32_t m_drops; /* times failed to find space */ 1115 u_int32_t m_wait; /* times waited for space */ 1116 u_int32_t m_drain; /* times drained protocols for space */ 1117 u_short m_mtypes[256]; /* type specific mbuf allocations */ 1118 u_int32_t m_mcfail; /* times m_copym failed */ 1119 u_int32_t m_mpfail; /* times m_pullup failed */ 1120 u_int32_t m_msize; /* length of an mbuf */ 1121 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 1122 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 1123 u_int32_t m_mlen; /* length of data in an mbuf */ 1124 u_int32_t m_mhlen; /* length of data in a header mbuf */ 1125 u_int32_t m_bigclusters; /* clusters obtained from page pool */ 1126 u_int32_t m_bigclfree; /* free clusters */ 1127 u_int32_t m_bigmclbytes; /* length of an mbuf cluster */ 1128 u_int32_t m_forcedefunct; /* times we force defunct'ed an app's sockets */ 1129 }; 1130 1131 /* Compatibillity with 10.3 */ 1132 struct ombstat { 1133 u_int32_t m_mbufs; /* mbufs obtained from page pool */ 1134 u_int32_t m_clusters; /* clusters obtained from page pool */ 1135 u_int32_t m_spare; /* spare field */ 1136 u_int32_t m_clfree; /* free clusters */ 1137 u_int32_t m_drops; /* times failed to find space */ 1138 u_int32_t m_wait; /* times waited for space */ 1139 u_int32_t m_drain; /* times drained protocols for space */ 1140 u_short m_mtypes[256]; /* type specific mbuf allocations */ 1141 u_int32_t m_mcfail; /* times m_copym failed */ 1142 u_int32_t m_mpfail; /* times m_pullup failed */ 1143 u_int32_t m_msize; /* length of an mbuf */ 1144 u_int32_t m_mclbytes; /* length of an mbuf cluster */ 1145 u_int32_t m_minclsize; /* min length of data to allocate a cluster */ 1146 u_int32_t m_mlen; /* length of data in an mbuf */ 1147 u_int32_t m_mhlen; /* length of data in a header mbuf */ 1148 }; 1149 1150 /* 1151 * mbuf class statistics. 1152 */ 1153 #define MAX_MBUF_CNAME 15 1154 1155 #if defined(XNU_KERNEL_PRIVATE) 1156 /* For backwards compatibility with 32-bit userland process */ 1157 struct omb_class_stat { 1158 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 1159 u_int32_t mbcl_size; /* buffer size */ 1160 u_int32_t mbcl_total; /* # of buffers created */ 1161 u_int32_t mbcl_active; /* # of active buffers */ 1162 u_int32_t mbcl_infree; /* # of available buffers */ 1163 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 1164 u_int32_t mbcl_pad; /* padding */ 1165 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 1166 u_int64_t mbcl_free_cnt; /* # of times free is called */ 1167 u_int64_t mbcl_notified; /* # of notified wakeups */ 1168 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 1169 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 1170 u_int32_t mbcl_ctotal; /* total only for this class */ 1171 u_int32_t mbcl_release_cnt; /* amount of memory returned */ 1172 /* 1173 * Cache layer statistics 1174 */ 1175 u_int32_t mbcl_mc_state; /* cache state (see below) */ 1176 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 1177 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 1178 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 1179 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 1180 u_int32_t mbcl_peak_reported; /* last usage peak reported */ 1181 u_int32_t mbcl_reserved[7]; /* for future use */ 1182 u_int32_t mbcl_pad2; /* padding */ 1183 } __attribute__((__packed__)); 1184 #endif /* XNU_KERNEL_PRIVATE */ 1185 1186 typedef struct mb_class_stat { 1187 char mbcl_cname[MAX_MBUF_CNAME + 1]; /* class name */ 1188 u_int32_t mbcl_size; /* buffer size */ 1189 u_int32_t mbcl_total; /* # of buffers created */ 1190 u_int32_t mbcl_active; /* # of active buffers */ 1191 u_int32_t mbcl_infree; /* # of available buffers */ 1192 u_int32_t mbcl_slab_cnt; /* # of available slabs */ 1193 #if defined(KERNEL) || defined(__LP64__) 1194 u_int32_t mbcl_pad; /* padding */ 1195 #endif /* KERNEL || __LP64__ */ 1196 u_int64_t mbcl_alloc_cnt; /* # of times alloc is called */ 1197 u_int64_t mbcl_free_cnt; /* # of times free is called */ 1198 u_int64_t mbcl_notified; /* # of notified wakeups */ 1199 u_int64_t mbcl_purge_cnt; /* # of purges so far */ 1200 u_int64_t mbcl_fail_cnt; /* # of allocation failures */ 1201 u_int32_t mbcl_ctotal; /* total only for this class */ 1202 u_int32_t mbcl_release_cnt; /* amount of memory returned */ 1203 /* 1204 * Cache layer statistics 1205 */ 1206 u_int32_t mbcl_mc_state; /* cache state (see below) */ 1207 u_int32_t mbcl_mc_cached; /* # of cached buffers */ 1208 u_int32_t mbcl_mc_waiter_cnt; /* # waiters on the cache */ 1209 u_int32_t mbcl_mc_wretry_cnt; /* # of wait retries */ 1210 u_int32_t mbcl_mc_nwretry_cnt; /* # of no-wait retry attempts */ 1211 u_int32_t mbcl_peak_reported; /* last usage peak reported */ 1212 u_int32_t mbcl_reserved[7]; /* for future use */ 1213 } mb_class_stat_t; 1214 1215 #define MCS_DISABLED 0 /* cache is permanently disabled */ 1216 #define MCS_ONLINE 1 /* cache is online */ 1217 #define MCS_PURGING 2 /* cache is being purged */ 1218 #define MCS_OFFLINE 3 /* cache is offline (resizing) */ 1219 1220 #if defined(XNU_KERNEL_PRIVATE) 1221 /* For backwards compatibility with 32-bit userland process */ 1222 struct omb_stat { 1223 u_int32_t mbs_cnt; /* number of classes */ 1224 u_int32_t mbs_pad; /* padding */ 1225 struct omb_class_stat mbs_class[1]; /* class array */ 1226 } __attribute__((__packed__)); 1227 #endif /* XNU_KERNEL_PRIVATE */ 1228 1229 typedef struct mb_stat { 1230 u_int32_t mbs_cnt; /* number of classes */ 1231 #if defined(KERNEL) || defined(__LP64__) 1232 u_int32_t mbs_pad; /* padding */ 1233 #endif /* KERNEL || __LP64__ */ 1234 mb_class_stat_t mbs_class[1]; /* class array */ 1235 } mb_stat_t; 1236 1237 #ifdef PRIVATE 1238 #define MLEAK_STACK_DEPTH 16 /* Max PC stack depth */ 1239 1240 typedef struct mleak_trace_stat { 1241 u_int64_t mltr_collisions; 1242 u_int64_t mltr_hitcount; 1243 u_int64_t mltr_allocs; 1244 u_int64_t mltr_depth; 1245 u_int64_t mltr_addr[MLEAK_STACK_DEPTH]; 1246 } mleak_trace_stat_t; 1247 1248 typedef struct mleak_stat { 1249 u_int32_t ml_isaddr64; /* 64-bit KVA? */ 1250 u_int32_t ml_cnt; /* number of traces */ 1251 mleak_trace_stat_t ml_trace[1]; /* trace array */ 1252 } mleak_stat_t; 1253 1254 struct mleak_table { 1255 u_int32_t mleak_capture; /* sampling capture counter */ 1256 u_int32_t mleak_sample_factor; /* sample factor */ 1257 1258 /* Times two active records want to occupy the same spot */ 1259 u_int64_t alloc_collisions; 1260 u_int64_t trace_collisions; 1261 1262 /* Times new record lands on spot previously occupied by freed alloc */ 1263 u_int64_t alloc_overwrites; 1264 u_int64_t trace_overwrites; 1265 1266 /* Times a new alloc or trace is put into the hash table */ 1267 u_int64_t alloc_recorded; 1268 u_int64_t trace_recorded; 1269 1270 /* Total number of outstanding allocs */ 1271 u_int64_t outstanding_allocs; 1272 1273 /* Times mleak_log returned false because couldn't acquire the lock */ 1274 u_int64_t total_conflicts; 1275 }; 1276 #endif /* PRIVATE */ 1277 1278 #ifdef KERNEL_PRIVATE 1279 __BEGIN_DECLS 1280 1281 /* 1282 * Exported (private) 1283 */ 1284 1285 extern struct mbstat mbstat; /* statistics */ 1286 1287 __END_DECLS 1288 #endif /* KERNEL_PRIVATE */ 1289 1290 #ifdef XNU_KERNEL_PRIVATE 1291 __BEGIN_DECLS 1292 1293 /* 1294 * Not exported (xnu private) 1295 */ 1296 1297 /* flags to m_get/MGET */ 1298 /* Need to include malloc.h to get right options for malloc */ 1299 #include <sys/malloc.h> 1300 1301 struct mbuf; 1302 1303 /* length to m_copy to copy all */ 1304 #define M_COPYALL 1000000000 1305 1306 #define M_DONTWAIT M_NOWAIT 1307 #define M_WAIT M_WAITOK 1308 1309 /* modes for m_copym and variants */ 1310 #define M_COPYM_NOOP_HDR 0 /* don't copy/move pkthdr contents */ 1311 #define M_COPYM_COPY_HDR 1 /* copy pkthdr from old to new */ 1312 #define M_COPYM_MOVE_HDR 2 /* move pkthdr from old to new */ 1313 #define M_COPYM_MUST_COPY_HDR 3 /* MUST copy pkthdr from old to new */ 1314 #define M_COPYM_MUST_MOVE_HDR 4 /* MUST move pkthdr from old to new */ 1315 1316 extern void m_freem(struct mbuf *) __XNU_INTERNAL(m_freem); 1317 extern u_int64_t mcl_to_paddr(char *); 1318 extern void m_adj(struct mbuf *, int); 1319 extern void m_cat(struct mbuf *, struct mbuf *); 1320 extern void m_copydata(struct mbuf *, int, int, void *); 1321 extern struct mbuf *m_copym(struct mbuf *, int, int, int); 1322 extern struct mbuf *m_copym_mode(struct mbuf *, int, int, int, uint32_t); 1323 extern struct mbuf *m_get(int, int); 1324 extern struct mbuf *m_gethdr(int, int); 1325 extern struct mbuf *m_getpacket(void); 1326 extern struct mbuf *m_getpackets(int, int, int); 1327 extern struct mbuf *m_mclget(struct mbuf *, int); 1328 extern void *m_mtod(struct mbuf *); 1329 extern struct mbuf *m_prepend_2(struct mbuf *, int, int, int); 1330 extern struct mbuf *m_pullup(struct mbuf *, int); 1331 extern struct mbuf *m_split(struct mbuf *, int, int); 1332 extern void m_mclfree(caddr_t p); 1333 extern int mbuf_get_class(struct mbuf *m); 1334 extern bool mbuf_class_under_pressure(struct mbuf *m); 1335 1336 /* 1337 * On platforms which require strict alignment (currently for anything but 1338 * i386 or x86_64), this macro checks whether the data pointer of an mbuf 1339 * is 32-bit aligned (this is the expected minimum alignment for protocol 1340 * headers), and assert otherwise. 1341 */ 1342 #if defined(__i386__) || defined(__x86_64__) 1343 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) 1344 #else /* !__i386__ && !__x86_64__ */ 1345 #define MBUF_STRICT_DATA_ALIGNMENT_CHECK_32(_m) do { \ 1346 if (!IS_P2ALIGNED((_m)->m_data, sizeof (u_int32_t))) { \ 1347 if (((_m)->m_flags & M_PKTHDR) && \ 1348 (_m)->m_pkthdr.rcvif != NULL) { \ 1349 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1350 "32-bit aligned [%s: alignerrs=%lld]\n", \ 1351 __func__, (_m), (_m)->m_data, \ 1352 if_name((_m)->m_pkthdr.rcvif), \ 1353 (_m)->m_pkthdr.rcvif->if_alignerrs); \ 1354 } else { \ 1355 panic_plain("\n%s: mbuf %p data ptr %p is not " \ 1356 "32-bit aligned\n", \ 1357 __func__, (_m), (_m)->m_data); \ 1358 } \ 1359 } \ 1360 } while (0) 1361 #endif /* !__i386__ && !__x86_64__ */ 1362 1363 /* Maximum number of MBUF_SC values (excluding MBUF_SC_UNSPEC) */ 1364 #define MBUF_SC_MAX_CLASSES 10 1365 1366 /* 1367 * These conversion macros rely on the corresponding MBUF_SC and 1368 * MBUF_TC values in order to establish the following mapping: 1369 * 1370 * MBUF_SC_BK_SYS ] ==> MBUF_TC_BK 1371 * MBUF_SC_BK ] 1372 * 1373 * MBUF_SC_BE ] ==> MBUF_TC_BE 1374 * MBUF_SC_RD ] 1375 * MBUF_SC_OAM ] 1376 * 1377 * MBUF_SC_AV ] ==> MBUF_TC_VI 1378 * MBUF_SC_RV ] 1379 * MBUF_SC_VI ] 1380 * MBUF_SC_SIG ] 1381 * 1382 * MBUF_SC_VO ] ==> MBUF_TC_VO 1383 * MBUF_SC_CTL ] 1384 * 1385 * The values assigned to each service class allows for a fast mapping to 1386 * the corresponding MBUF_TC traffic class values, as well as to retrieve the 1387 * assigned index; therefore care must be taken when comparing against these 1388 * values. Use the corresponding class and index macros to retrieve the 1389 * corresponding portion, and never assume that a higher class corresponds 1390 * to a higher index. 1391 */ 1392 #define MBUF_SCVAL(x) ((x) & 0xffff) 1393 #define MBUF_SCIDX(x) ((((x) >> 16) & 0xff) >> 3) 1394 #define MBUF_SC2TC(_sc) (MBUF_SCVAL(_sc) >> 7) 1395 #define MBUF_TC2SCVAL(_tc) ((_tc) << 7) 1396 #define IS_MBUF_SC_BACKGROUND(_sc) (((_sc) == MBUF_SC_BK_SYS) || \ 1397 ((_sc) == MBUF_SC_BK)) 1398 #define IS_MBUF_SC_REALTIME(_sc) ((_sc) >= MBUF_SC_AV && (_sc) <= MBUF_SC_VO) 1399 #define IS_MBUF_SC_BESTEFFORT(_sc) ((_sc) == MBUF_SC_BE || \ 1400 (_sc) == MBUF_SC_RD || (_sc) == MBUF_SC_OAM) 1401 1402 #define SCIDX_BK_SYS MBUF_SCIDX(MBUF_SC_BK_SYS) 1403 #define SCIDX_BK MBUF_SCIDX(MBUF_SC_BK) 1404 #define SCIDX_BE MBUF_SCIDX(MBUF_SC_BE) 1405 #define SCIDX_RD MBUF_SCIDX(MBUF_SC_RD) 1406 #define SCIDX_OAM MBUF_SCIDX(MBUF_SC_OAM) 1407 #define SCIDX_AV MBUF_SCIDX(MBUF_SC_AV) 1408 #define SCIDX_RV MBUF_SCIDX(MBUF_SC_RV) 1409 #define SCIDX_VI MBUF_SCIDX(MBUF_SC_VI) 1410 #define SCIDX_SIG MBUF_SCIDX(MBUF_SC_SIG) 1411 #define SCIDX_VO MBUF_SCIDX(MBUF_SC_VO) 1412 #define SCIDX_CTL MBUF_SCIDX(MBUF_SC_CTL) 1413 1414 #define SCVAL_BK_SYS MBUF_SCVAL(MBUF_SC_BK_SYS) 1415 #define SCVAL_BK MBUF_SCVAL(MBUF_SC_BK) 1416 #define SCVAL_BE MBUF_SCVAL(MBUF_SC_BE) 1417 #define SCVAL_RD MBUF_SCVAL(MBUF_SC_RD) 1418 #define SCVAL_OAM MBUF_SCVAL(MBUF_SC_OAM) 1419 #define SCVAL_AV MBUF_SCVAL(MBUF_SC_AV) 1420 #define SCVAL_RV MBUF_SCVAL(MBUF_SC_RV) 1421 #define SCVAL_VI MBUF_SCVAL(MBUF_SC_VI) 1422 #define SCVAL_SIG MBUF_SCVAL(MBUF_SC_SIG) 1423 #define SCVAL_VO MBUF_SCVAL(MBUF_SC_VO) 1424 #define SCVAL_CTL MBUF_SCVAL(MBUF_SC_CTL) 1425 1426 #define MBUF_VALID_SC(c) \ 1427 (c == MBUF_SC_BK_SYS || c == MBUF_SC_BK || c == MBUF_SC_BE || \ 1428 c == MBUF_SC_RD || c == MBUF_SC_OAM || c == MBUF_SC_AV || \ 1429 c == MBUF_SC_RV || c == MBUF_SC_VI || c == MBUF_SC_SIG || \ 1430 c == MBUF_SC_VO || c == MBUF_SC_CTL) 1431 1432 #define MBUF_VALID_SCIDX(c) \ 1433 (c == SCIDX_BK_SYS || c == SCIDX_BK || c == SCIDX_BE || \ 1434 c == SCIDX_RD || c == SCIDX_OAM || c == SCIDX_AV || \ 1435 c == SCIDX_RV || c == SCIDX_VI || c == SCIDX_SIG || \ 1436 c == SCIDX_VO || c == SCIDX_CTL) 1437 1438 #define MBUF_VALID_SCVAL(c) \ 1439 (c == SCVAL_BK_SYS || c == SCVAL_BK || c == SCVAL_BE || \ 1440 c == SCVAL_RD || c == SCVAL_OAM || c == SCVAL_AV || \ 1441 c == SCVAL_RV || c == SCVAL_VI || c == SCVAL_SIG || \ 1442 c == SCVAL_VO || SCVAL_CTL) 1443 1444 extern unsigned char *mbutl; /* start VA of mbuf pool */ 1445 extern unsigned char *embutl; /* end VA of mbuf pool */ 1446 extern unsigned int nmbclusters; /* number of mapped clusters */ 1447 extern int njcl; /* # of jumbo clusters */ 1448 extern int njclbytes; /* size of a jumbo cluster */ 1449 extern int max_hdr; /* largest link+protocol header */ 1450 extern int max_datalen; /* MHLEN - max_hdr */ 1451 1452 /* Use max_linkhdr instead of _max_linkhdr */ 1453 extern int _max_linkhdr; /* largest link-level header */ 1454 1455 /* Use max_protohdr instead of _max_protohdr */ 1456 extern int _max_protohdr; /* largest protocol header */ 1457 1458 __private_extern__ unsigned int mbuf_default_ncl(uint64_t); 1459 __private_extern__ void mbinit(void); 1460 __private_extern__ struct mbuf *m_clattach(struct mbuf *, int, caddr_t, 1461 void (*)(caddr_t, u_int, caddr_t), size_t, caddr_t, int, int); 1462 __private_extern__ caddr_t m_bigalloc(int); 1463 __private_extern__ void m_bigfree(caddr_t, u_int, caddr_t); 1464 __private_extern__ struct mbuf *m_mbigget(struct mbuf *, int); 1465 __private_extern__ caddr_t m_16kalloc(int); 1466 __private_extern__ void m_16kfree(caddr_t, u_int, caddr_t); 1467 __private_extern__ struct mbuf *m_m16kget(struct mbuf *, int); 1468 __private_extern__ int m_reinit(struct mbuf *, int); 1469 __private_extern__ struct mbuf *m_free(struct mbuf *) __XNU_INTERNAL(m_free); 1470 __private_extern__ struct mbuf *m_getclr(int, int); 1471 __private_extern__ struct mbuf *m_getptr(struct mbuf *, int, int *); 1472 __private_extern__ unsigned int m_length(struct mbuf *); 1473 __private_extern__ unsigned int m_length2(struct mbuf *, struct mbuf **); 1474 __private_extern__ unsigned int m_fixhdr(struct mbuf *); 1475 __private_extern__ struct mbuf *m_defrag(struct mbuf *, int); 1476 __private_extern__ struct mbuf *m_defrag_offset(struct mbuf *, u_int32_t, int); 1477 __private_extern__ struct mbuf *m_prepend(struct mbuf *, int, int); 1478 __private_extern__ struct mbuf *m_copyup(struct mbuf *, int, int); 1479 __private_extern__ struct mbuf *m_retry(int, int); 1480 __private_extern__ struct mbuf *m_retryhdr(int, int); 1481 __private_extern__ int m_freem_list(struct mbuf *); 1482 __private_extern__ int m_append(struct mbuf *, int, caddr_t); 1483 __private_extern__ struct mbuf *m_last(struct mbuf *); 1484 __private_extern__ struct mbuf *m_devget(char *, int, int, struct ifnet *, 1485 void (*)(const void *, void *, size_t)); 1486 __private_extern__ struct mbuf *m_pulldown(struct mbuf *, int, int, int *); 1487 1488 __private_extern__ struct mbuf *m_getcl(int, int, int); 1489 __private_extern__ caddr_t m_mclalloc(int); 1490 __private_extern__ int m_mclhasreference(struct mbuf *); 1491 __private_extern__ void m_copy_pkthdr(struct mbuf *, struct mbuf *); 1492 __private_extern__ int m_dup_pkthdr(struct mbuf *, struct mbuf *, int); 1493 __private_extern__ void m_copy_pftag(struct mbuf *, struct mbuf *); 1494 __private_extern__ void m_copy_necptag(struct mbuf *, struct mbuf *); 1495 __private_extern__ void m_copy_classifier(struct mbuf *, struct mbuf *); 1496 1497 __private_extern__ struct mbuf *m_dtom(void *); 1498 __private_extern__ int m_mtocl(void *); 1499 __private_extern__ union mcluster *m_cltom(int); 1500 1501 __private_extern__ void m_align(struct mbuf *, int); 1502 1503 __private_extern__ struct mbuf *m_normalize(struct mbuf *m); 1504 __private_extern__ void m_mchtype(struct mbuf *m, int t); 1505 __private_extern__ void m_mcheck(struct mbuf *); 1506 1507 __private_extern__ void m_copyback(struct mbuf *, int, int, const void *); 1508 __private_extern__ struct mbuf *m_copyback_cow(struct mbuf *, int, int, 1509 const void *, int); 1510 __private_extern__ int m_makewritable(struct mbuf **, int, int, int); 1511 __private_extern__ struct mbuf *m_dup(struct mbuf *m, int how); 1512 __private_extern__ struct mbuf *m_copym_with_hdrs(struct mbuf *, int, int, int, 1513 struct mbuf **, int *, uint32_t); 1514 __private_extern__ struct mbuf *m_getpackethdrs(int, int); 1515 __private_extern__ struct mbuf *m_getpacket_how(int); 1516 __private_extern__ struct mbuf *m_getpackets_internal(unsigned int *, int, 1517 int, int, size_t); 1518 __private_extern__ struct mbuf *m_allocpacket_internal(unsigned int *, size_t, 1519 unsigned int *, int, int, size_t); 1520 1521 __private_extern__ int m_ext_set_prop(struct mbuf *, uint32_t, uint32_t); 1522 __private_extern__ uint32_t m_ext_get_prop(struct mbuf *); 1523 __private_extern__ int m_ext_paired_is_active(struct mbuf *); 1524 __private_extern__ void m_ext_paired_activate(struct mbuf *); 1525 1526 __private_extern__ void m_add_crumb(struct mbuf *, uint16_t); 1527 1528 __private_extern__ void mbuf_drain(boolean_t); 1529 1530 /* 1531 * Packets may have annotations attached by affixing a list of "packet 1532 * tags" to the pkthdr structure. Packet tags are dynamically allocated 1533 * semi-opaque data structures that have a fixed header (struct m_tag) 1534 * that specifies the size of the memory block and an <id,type> pair that 1535 * identifies it. The id identifies the module and the type identifies the 1536 * type of data for that module. The id of zero is reserved for the kernel. 1537 * 1538 * Note that the packet tag returned by m_tag_allocate has the default 1539 * memory alignment implemented by malloc. To reference private data one 1540 * can use a construct like: 1541 * 1542 * struct m_tag *mtag = m_tag_allocate(...); 1543 * struct foo *p = (struct foo *)(mtag+1); 1544 * 1545 * if the alignment of struct m_tag is sufficient for referencing members 1546 * of struct foo. Otherwise it is necessary to embed struct m_tag within 1547 * the private data structure to insure proper alignment; e.g. 1548 * 1549 * struct foo { 1550 * struct m_tag tag; 1551 * ... 1552 * }; 1553 * struct foo *p = (struct foo *) m_tag_allocate(...); 1554 * struct m_tag *mtag = &p->tag; 1555 */ 1556 1557 #define KERNEL_MODULE_TAG_ID 0 1558 1559 enum { 1560 KERNEL_TAG_TYPE_NONE = 0, 1561 KERNEL_TAG_TYPE_DUMMYNET = 1, 1562 KERNEL_TAG_TYPE_DIVERT = 2, 1563 KERNEL_TAG_TYPE_IPFORWARD = 3, 1564 KERNEL_TAG_TYPE_IPFILT = 4, 1565 KERNEL_TAG_TYPE_MACLABEL = 5, 1566 KERNEL_TAG_TYPE_MAC_POLICY_LABEL = 6, 1567 KERNEL_TAG_TYPE_ENCAP = 8, 1568 KERNEL_TAG_TYPE_INET6 = 9, 1569 KERNEL_TAG_TYPE_IPSEC = 10, 1570 KERNEL_TAG_TYPE_DRVAUX = 11, 1571 #if SKYWALK 1572 KERNEL_TAG_TYPE_PKT_FLOWADV = 12, 1573 #endif /* SKYWALK */ 1574 KERNEL_TAG_TYPE_CFIL_UDP = 13, 1575 KERNEL_TAG_TYPE_PF_REASS = 14, 1576 }; 1577 1578 /* Packet tag routines */ 1579 __private_extern__ struct m_tag *m_tag_alloc(u_int32_t, u_int16_t, int, int); 1580 __private_extern__ struct m_tag *m_tag_create(u_int32_t, u_int16_t, int, int, 1581 struct mbuf *); 1582 __private_extern__ void m_tag_free(struct m_tag *); 1583 __private_extern__ void m_tag_prepend(struct mbuf *, struct m_tag *); 1584 __private_extern__ void m_tag_unlink(struct mbuf *, struct m_tag *); 1585 __private_extern__ void m_tag_delete(struct mbuf *, struct m_tag *); 1586 __private_extern__ void m_tag_delete_chain(struct mbuf *, struct m_tag *); 1587 __private_extern__ struct m_tag *m_tag_locate(struct mbuf *, u_int32_t, 1588 u_int16_t, struct m_tag *); 1589 __private_extern__ struct m_tag *m_tag_copy(struct m_tag *, int); 1590 __private_extern__ int m_tag_copy_chain(struct mbuf *, struct mbuf *, int); 1591 __private_extern__ void m_tag_init(struct mbuf *, int); 1592 __private_extern__ struct m_tag *m_tag_first(struct mbuf *); 1593 __private_extern__ struct m_tag *m_tag_next(struct mbuf *, struct m_tag *); 1594 1595 __private_extern__ void m_scratch_init(struct mbuf *); 1596 __private_extern__ u_int32_t m_scratch_get(struct mbuf *, u_int8_t **); 1597 1598 __private_extern__ void m_classifier_init(struct mbuf *, uint32_t); 1599 1600 __private_extern__ int m_set_service_class(struct mbuf *, mbuf_svc_class_t); 1601 __private_extern__ mbuf_svc_class_t m_get_service_class(struct mbuf *); 1602 __private_extern__ mbuf_svc_class_t m_service_class_from_idx(u_int32_t); 1603 __private_extern__ mbuf_svc_class_t m_service_class_from_val(u_int32_t); 1604 __private_extern__ int m_set_traffic_class(struct mbuf *, mbuf_traffic_class_t); 1605 __private_extern__ mbuf_traffic_class_t m_get_traffic_class(struct mbuf *); 1606 1607 #define ADDCARRY(_x) do { \ 1608 while (((_x) >> 16) != 0) \ 1609 (_x) = ((_x) >> 16) + ((_x) & 0xffff); \ 1610 } while (0) 1611 1612 __private_extern__ u_int16_t m_adj_sum16(struct mbuf *, u_int32_t, 1613 u_int32_t, u_int32_t, u_int32_t); 1614 __private_extern__ u_int16_t m_sum16(struct mbuf *, u_int32_t, u_int32_t); 1615 1616 __private_extern__ void m_set_ext(struct mbuf *, struct ext_ref *, 1617 m_ext_free_func_t, caddr_t); 1618 __private_extern__ struct ext_ref *m_get_rfa(struct mbuf *); 1619 __private_extern__ m_ext_free_func_t m_get_ext_free(struct mbuf *); 1620 __private_extern__ caddr_t m_get_ext_arg(struct mbuf *); 1621 1622 __private_extern__ void m_do_tx_compl_callback(struct mbuf *, struct ifnet *); 1623 __private_extern__ mbuf_tx_compl_func m_get_tx_compl_callback(u_int32_t); 1624 1625 __END_DECLS 1626 #endif /* XNU_KERNEL_PRIVATE */ 1627 #endif /* !_SYS_MBUF_H_ */ 1628