1 /*
2 * Copyright (c) 2016-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #ifndef _SKYWALK_PACKET_PACKETVAR_H_
30 #define _SKYWALK_PACKET_PACKETVAR_H_
31
32 #ifdef BSD_KERNEL_PRIVATE
33 #include <skywalk/core/skywalk_var.h>
34 #include <skywalk/os_packet_private.h>
35
36 /*
37 * Kernel variant of __user_buflet.
38 *
39 * The main difference here is the support for shared buffers, where
40 * multiple buflets may point to the same buffer object at different
41 * data span within it, each holding a reference to the buffer object,
42 * i.e. the "use" count. The buf_addr therefore points to the beginning
43 * of the data span; the buf_len describes the length of the span; and
44 * the buf_doff describes the offset relative to the beginning of the
45 * span as noted by buf_addr. The buffer object is stored in buf_objaddr.
46 */
47 struct __kern_buflet {
48 /*
49 * Common area between user and kernel variants.
50 */
51 struct __buflet buf_com;
52 /*
53 * Kernel specific.
54 */
55 /* buffer control of the buffer object */
56 const struct skmem_bufctl *buf_ctl;
57
58 #define buf_objaddr buf_ctl->bc_addr
59 #define buf_objlim buf_ctl->bc_lim
60 } __attribute((packed));
61
62 struct __kern_buflet_ext {
63 /*
64 * This is an overlay structure on __kern_buflet.
65 */
66 struct __kern_buflet kbe_overlay;
67 /*
68 * extended variant specific.
69 */
70 /* mirrored user buflet */
71 struct __user_buflet const *kbe_buf_user;
72
73 /* buflet user packet pool hash bucket linkage */
74 SLIST_ENTRY(__kern_buflet_ext) kbe_buf_upp_link;
75
76 /* pid of the process using the buflet */
77 pid_t kbe_buf_pid;
78 } __attribute((packed));
79
80 #define KBUF_CTOR(_kbuf, _baddr, _bidxreg, _bc, _pp, _large) do { \
81 _CASSERT(sizeof ((_kbuf)->buf_addr) == sizeof (mach_vm_address_t));\
82 /* kernel variant (deconst) */ \
83 BUF_CTOR(_kbuf, _baddr, _bidxreg, (_large) ? PP_BUF_SIZE_LARGE(_pp) :\
84 PP_BUF_SIZE_DEF(_pp), 0, 0, (_kbuf)->buf_nbft_addr, \
85 (_kbuf)->buf_nbft_idx, (_kbuf)->buf_flag, 0, 0); \
86 *(struct skmem_bufctl **)(uintptr_t)&(_kbuf)->buf_ctl = (_bc); \
87 /* this may be called to initialize unused buflets */ \
88 if (__probable((_bc) != NULL)) { \
89 skmem_bufctl_use(_bc); \
90 } \
91 /* no need to construct user variant as it is done in externalize */ \
92 } while (0)
93
94 #define KBUF_EXT_CTOR(_kbuf, _ubuf, _baddr, _bidxreg, _bc, \
95 _bft_idx_reg, _pp, _large, _attch_buf) do { \
96 ASSERT(_bft_idx_reg != OBJ_IDX_NONE); \
97 _CASSERT(sizeof((_kbuf)->buf_flag) == sizeof(uint16_t)); \
98 /* we don't set buf_nbft_addr here as during construction it */ \
99 /* is used by skmem batch alloc logic */ \
100 *__DECONST(uint16_t *, &(_kbuf)->buf_flag) = BUFLET_FLAG_EXTERNAL;\
101 if (_large) { \
102 *__DECONST(uint16_t *, &(_kbuf)->buf_flag) |= \
103 BUFLET_FLAG_LARGE_BUF; \
104 } \
105 if (!_attch_buf) { \
106 *__DECONST(uint16_t *, &(_kbuf)->buf_flag) |= \
107 BUFLET_FLAG_RAW; \
108 } \
109 BUF_NBFT_IDX(_kbuf, OBJ_IDX_NONE); \
110 BUF_BFT_IDX_REG(_kbuf, _bft_idx_reg); \
111 *__DECONST(struct __user_buflet **, \
112 &((struct __kern_buflet_ext *)(_kbuf))->kbe_buf_user) = (_ubuf);\
113 KBUF_CTOR(_kbuf, _baddr, _bidxreg, _bc, _pp, _large); \
114 } while (0)
115
116 #define KBUF_INIT(_kbuf) do { \
117 ASSERT((_kbuf)->buf_ctl != NULL); \
118 ASSERT((_kbuf)->buf_addr != 0); \
119 ASSERT((_kbuf)->buf_dlim != 0); \
120 /* kernel variant (deconst) */ \
121 BUF_INIT(_kbuf, 0, 0); \
122 } while (0)
123
124 #define KBUF_EXT_INIT(_kbuf, _pp) do { \
125 _CASSERT(sizeof((_kbuf)->buf_boff) == sizeof(uint16_t)); \
126 _CASSERT(sizeof((_kbuf)->buf_grolen) == sizeof(uint16_t)); \
127 _CASSERT(sizeof((_kbuf)->buf_dlim) == sizeof(uint16_t)); \
128 ASSERT((_kbuf)->buf_ctl != NULL); \
129 ASSERT((_kbuf)->buf_flag & BUFLET_FLAG_EXTERNAL); \
130 ASSERT(((_kbuf)->buf_flag & BUFLET_FLAG_RAW) == 0); \
131 ASSERT((_kbuf)->buf_bft_idx_reg != OBJ_IDX_NONE); \
132 BUF_BADDR(_kbuf, (_kbuf)->buf_ctl->bc_addr); \
133 BUF_NBFT_ADDR(_kbuf, 0); \
134 BUF_NBFT_IDX(_kbuf, OBJ_IDX_NONE); \
135 *__DECONST(uint16_t *, &(_kbuf)->buf_dlim) = \
136 BUFLET_HAS_LARGE_BUF(_kbuf) ? PP_BUF_SIZE_LARGE((_pp)) : \
137 PP_BUF_SIZE_DEF((_pp)); \
138 (_kbuf)->buf_dlen = 0; \
139 (_kbuf)->buf_doff = 0; \
140 *__DECONST(uint16_t *, &(_kbuf)->buf_boff) = 0; \
141 *__DECONST(uint16_t *, &(_kbuf)->buf_grolen) = 0; \
142 ((struct __kern_buflet_ext *)(_kbuf))->kbe_buf_pid = (pid_t)-1; \
143 ((struct __kern_buflet_ext *)(_kbuf))->kbe_buf_upp_link.sle_next = NULL;\
144 } while (0)
145
146 #define RAW_KBUF_EXT_INIT(_kbuf) do { \
147 _CASSERT(sizeof((_kbuf)->buf_dlim) == sizeof(uint16_t)); \
148 _CASSERT(sizeof((_kbuf)->buf_grolen) == sizeof(uint16_t)); \
149 _CASSERT(sizeof((_kbuf)->buf_boff) == sizeof(uint16_t)); \
150 _CASSERT(sizeof((_kbuf)->buf_flag) == sizeof(uint16_t)); \
151 ASSERT((_kbuf)->buf_flag & BUFLET_FLAG_EXTERNAL); \
152 ASSERT((_kbuf)->buf_flag & BUFLET_FLAG_RAW); \
153 ASSERT((_kbuf)->buf_bft_idx_reg != OBJ_IDX_NONE); \
154 BUF_BADDR(_kbuf, 0); \
155 BUF_BIDX(_kbuf, OBJ_IDX_NONE); \
156 BUF_NBFT_ADDR(_kbuf, 0); \
157 BUF_NBFT_IDX(_kbuf, OBJ_IDX_NONE); \
158 *__DECONST(uint16_t *, &(_kbuf)->buf_dlim) = 0; \
159 *__DECONST(uint16_t *, &(_kbuf)->buf_grolen) = 0; \
160 *__DECONST(uint16_t *, &(_kbuf)->buf_boff) = 0; \
161 *__DECONST(uint16_t *, &(_kbuf)->buf_flag) &= ~BUFLET_FLAG_LARGE_BUF;\
162 (_kbuf)->buf_dlen = 0; \
163 (_kbuf)->buf_doff = 0; \
164 (_kbuf)->buf_ctl = NULL; \
165 ((struct __kern_buflet_ext *)(_kbuf))->kbe_buf_pid = (pid_t)-1; \
166 ((struct __kern_buflet_ext *)(_kbuf))->kbe_buf_upp_link.sle_next = NULL;\
167 } while (0)
168
169 /* initialize struct __user_buflet from struct __kern_buflet */
170 #define UBUF_INIT(_kbuf, _ubuf) do { \
171 BUF_CTOR(_ubuf, 0, (_kbuf)->buf_idx, (_kbuf)->buf_dlim, \
172 (_kbuf)->buf_dlen, (_kbuf)->buf_doff, (_kbuf)->buf_nbft_addr,\
173 (_kbuf)->buf_nbft_idx, (_kbuf)->buf_flag, \
174 (_kbuf)->buf_boff, (_kbuf)->buf_grolen); \
175 BUF_BFT_IDX_REG(_ubuf, (_kbuf)->buf_bft_idx_reg); \
176 } while (0)
177
178 #define KBUF_EXTERNALIZE(_kbuf, _ubuf, _pp) do { \
179 ASSERT(BUFLET_FROM_RAW_BFLT_CACHE(_kbuf) || \
180 (_kbuf)->buf_dlim == BUFLET_HAS_LARGE_BUF(_kbuf) ? \
181 PP_BUF_SIZE_LARGE((_pp)) : PP_BUF_SIZE_DEF((_pp))); \
182 ASSERT((_kbuf)->buf_addr != 0); \
183 /* For now, user-facing pool does not support shared */ \
184 /* buffer, since otherwise the ubuf and kbuf buffer */ \
185 /* indices would not match. Assert this is the case.*/ \
186 ASSERT((mach_vm_address_t)(_kbuf)->buf_objaddr + \
187 (_kbuf)->buf_boff == (_kbuf)->buf_addr); \
188 /* Initialize user buflet metadata from kernel buflet */ \
189 UBUF_INIT(_kbuf, _ubuf); \
190 } while (0)
191
192 #define KBUF_LINK(_pkbuf, _kbuf) do { \
193 ASSERT(__DECONST(void *, (_pkbuf)->buf_nbft_addr) == NULL); \
194 ASSERT(__DECONST(obj_idx_t, (_pkbuf)->buf_nbft_idx) == OBJ_IDX_NONE); \
195 ASSERT((_kbuf) != NULL); \
196 ASSERT((_kbuf)->buf_bft_idx_reg != OBJ_IDX_NONE); \
197 BUF_NBFT_ADDR(_pkbuf, _kbuf); \
198 BUF_NBFT_IDX(_pkbuf, (_kbuf)->buf_bft_idx_reg); \
199 } while (0)
200
201 #define KBUF_DTOR(_kbuf, _usecnt) do { \
202 if (__probable((_kbuf)->buf_ctl != NULL)) { \
203 (_usecnt) = skmem_bufctl_unuse( \
204 __DECONST(struct skmem_bufctl *, (_kbuf)->buf_ctl));\
205 *(struct skmem_bufctl **) \
206 (uintptr_t)&(_kbuf)->buf_ctl = NULL; \
207 } \
208 BUF_BADDR(_kbuf, 0); \
209 BUF_BIDX(_kbuf, OBJ_IDX_NONE); \
210 } while (0)
211
212 /*
213 * Copy kernel buflet (and add reference count to buffer).
214 */
215 #define _KBUF_COPY(_skb, _dkb) do { \
216 ASSERT((_skb)->buf_nbft_addr == 0); \
217 ASSERT((_skb)->buf_nbft_idx == OBJ_IDX_NONE); \
218 ASSERT(!((_dkb)->buf_flag & BUFLET_FLAG_EXTERNAL) || \
219 ((_dkb)->buf_flag & BUFLET_FLAG_RAW)); \
220 _CASSERT(sizeof(struct __kern_buflet) == 48); \
221 /* copy everything in the kernel buflet */ \
222 sk_copy64_40((uint64_t *)(void *)(_skb), (uint64_t *)(void *)(_dkb));\
223 ((uint64_t *)(void *)(_dkb))[5] = ((uint64_t *)(void *)(_skb))[5];\
224 ASSERT((_dkb)->buf_ctl == (_skb)->buf_ctl); \
225 if (__probable((_dkb)->buf_ctl != NULL)) { \
226 skmem_bufctl_use(__DECONST(struct skmem_bufctl *, \
227 (_dkb)->buf_ctl)); \
228 } \
229 } while (0)
230
231 /*
232 * Kernel variant of __user_quantum.
233 */
234 struct __kern_quantum {
235 /*
236 * Common area between user and kernel variants.
237 */
238 struct __quantum qum_com;
239
240 /*
241 * Kernel specific.
242 */
243 SLIST_ENTRY(__kern_quantum) qum_upp_link;
244 const struct kern_pbufpool *qum_pp;
245 const struct __user_quantum *qum_user;
246 const struct __kern_slot_desc *qum_ksd;
247 struct __kern_buflet qum_buf[1]; /* 1 buflet */
248 pid_t qum_pid;
249 } __attribute((aligned(sizeof(uint64_t))));
250
251 #define KQUM_CTOR(_kqum, _midx, _uqum, _pp, _qflags) do { \
252 ASSERT((uintptr_t)(_kqum) != (uintptr_t)(_uqum)); \
253 _CASSERT(sizeof(METADATA_IDX(_kqum)) == sizeof(obj_idx_t)); \
254 /* kernel variant (deconst) */ \
255 _KQUM_CTOR(_kqum, (PP_KERNEL_ONLY(_pp) ? \
256 QUM_F_KERNEL_ONLY : 0) | _qflags, 0, 0, OBJ_IDX_NONE, \
257 PP_BUF_SIZE_DEF((_pp)), _midx); \
258 _CASSERT(NEXUS_META_TYPE_MAX <= UINT16_MAX); \
259 METADATA_TYPE(_kqum) = (uint16_t)(_pp)->pp_md_type; \
260 _CASSERT(NEXUS_META_SUBTYPE_MAX <= UINT16_MAX); \
261 METADATA_SUBTYPE(_kqum) = (uint16_t)(_pp)->pp_md_subtype; \
262 *(struct kern_pbufpool **)(uintptr_t)&(_kqum)->qum_pp = (_pp); \
263 *(struct __user_quantum **)(uintptr_t)&(_kqum)->qum_user = (_uqum); \
264 *(obj_idx_t *)(uintptr_t)&METADATA_IDX(_kqum) = (_midx); \
265 (_kqum)->qum_pid = (pid_t)-1; \
266 *(struct __kern_slot_desc **)(uintptr_t)&(_kqum)->qum_ksd = NULL;\
267 /* no need to construct user variant as it is done in externalize */ \
268 } while (0)
269
270 #define KQUM_INIT(_kqum, _flags) do { \
271 ASSERT((_kqum)->qum_ksd == NULL); \
272 ASSERT((_kqum)->qum_pid == (pid_t)-1); \
273 /* kernel variant (deconst) */ \
274 _KQUM_INIT(_kqum, (PP_KERNEL_ONLY((_kqum)->qum_pp) ? \
275 QUM_F_KERNEL_ONLY : 0) | _flags, 0, METADATA_IDX(_kqum)); \
276 /* no need to initialize user variant as it is done in externalize */ \
277 } while (0)
278
279 __attribute__((always_inline))
280 inline boolean_t
_UUID_MATCH(uuid_t u1,uuid_t u2)281 _UUID_MATCH(uuid_t u1, uuid_t u2)
282 {
283 uint64_t *a = (uint64_t *)(void *) u1;
284 uint64_t *b = (uint64_t *)(void *) u2;
285 bool first_same = (a[0] == b[0]);
286 bool second_same = (a[1] == b[1]);
287
288 return first_same && second_same;
289 }
290
291 #define _UUID_COPY(_dst, _src) do { \
292 _CASSERT(sizeof (uuid_t) == 16); \
293 sk_copy64_16((uint64_t *)(void *)_src, (uint64_t *)(void *)_dst); \
294 } while (0)
295
296 #define _UUID_CLEAR(_u) do { \
297 uint64_t *__dst = (uint64_t *)(void *)(_u); \
298 _CASSERT(sizeof (uuid_t) == 16); \
299 *(__dst++) = 0; /* qw[0] */ \
300 *(__dst) = 0; /* qw[1] */ \
301 } while (0)
302
303 /*
304 * _QUM_COPY only copies the user metadata portion of the quantum;
305 * at the moment this is everything from the beginning down to __q_flags,
306 * but no more. It preserves the destination's QUM_F_SAVE_MASK bits.
307 *
308 * NOTE: this needs to be adjusted if more user-mutable field is added
309 * after __q_flags.
310 */
311 #define _QUM_COPY(_skq, _dkq) do { \
312 volatile uint16_t _sf = ((_dkq)->qum_qflags & QUM_F_SAVE_MASK); \
313 _CASSERT(sizeof (_sf) == sizeof ((_dkq)->qum_qflags)); \
314 _CASSERT(offsetof(struct __quantum, __q_flags) == 24); \
315 /* copy everything above (and excluding) __q_flags */ \
316 sk_copy64_24((uint64_t *)(void *)&(_skq)->qum_com, \
317 (uint64_t *)(void *)&(_dkq)->qum_com); \
318 /* copy __q_flags and restore saved bits */ \
319 (_dkq)->qum_qflags = ((_skq)->qum_qflags & ~QUM_F_SAVE_MASK) | _sf; \
320 } while (0)
321
322 /*
323 * _QUM_INTERNALIZE internalizes a portion of the quantum that includes
324 * user visible fields without overwriting the portion that's private to
325 * the kernel; see comments on _QUM_COPY().
326 */
327 #define _QUM_INTERNALIZE(_uq, _kq) do { \
328 _QUM_COPY(_uq, _kq); \
329 /* drop all but QUM_F_SAVE_MASK */ \
330 (_kq)->qum_qflags &= QUM_F_SAVE_MASK; \
331 } while (0)
332
333 /*
334 * _QUM_EXTERNALIZE externalizes a portion of the quantum that's user
335 * visible without including fields that's private to the kernel; at
336 * the moment this is everything from the begininng down to __q_flags,
337 * but no more. It does NOT preserve the destination's QUM_F_SAVE_MASK
338 * bits, but instead copies all bits except QUMF_KERNEL_FLAGS ones.
339 *
340 * NOTE: this needs to be adjusted if more user-mutable field is added
341 * after __q_flags. This macro is used only during externalize.
342 */
343 #define _QUM_EXTERNALIZE(_kq, _uq) do { \
344 _CASSERT(offsetof(struct __quantum, __q_flags) == 24); \
345 _CASSERT(sizeof(METADATA_IDX(_uq)) == sizeof(obj_idx_t)); \
346 /* copy __quantum excluding qum_qflags */ \
347 sk_copy64_24((uint64_t *)(void *)&(_kq)->qum_com, \
348 (uint64_t *)(void *)&(_uq)->qum_com); \
349 /* copy qum_qflags excluding saved bits */ \
350 (_uq)->qum_qflags = ((_kq)->qum_qflags & ~QUM_F_KERNEL_FLAGS); \
351 /* re-initialize user metadata */ \
352 *(obj_idx_t *)(uintptr_t)&METADATA_IDX(_uq) = METADATA_IDX(_kq); \
353 METADATA_TYPE(_uq) = METADATA_TYPE(_kq); \
354 METADATA_SUBTYPE(_uq) = METADATA_SUBTYPE(_kq); \
355 (_uq)->qum_usecnt = 0; \
356 } while (0)
357
358 /*
359 * Transmit completion.
360 */
361 struct __packet_compl {
362 /*
363 * Tx completion data
364 * _arg & _data: context data which are passed as arguments
365 * to the registered Tx completion callback.
366 * _tx_status: Tx status set by the driver.
367 */
368 union {
369 uint64_t compl_data64[3];
370 struct {
371 uintptr_t _cb_arg;
372 uintptr_t _cb_data;
373 uint32_t _tx_status;
374 uint32_t _pad;
375 } compl_data;
376 };
377 /* bitmap indicating the requested packet completion callbacks */
378 uint32_t compl_callbacks;
379 /* Context identifier for a given packet completion */
380 uint32_t compl_context;
381 };
382
383 /*
384 * Kernel variant of __user_packet.
385 */
386 struct __kern_packet {
387 struct __kern_quantum pkt_qum;
388 #define pkt_user pkt_qum.qum_user
389
390 /*
391 * Common area between user and kernel variants.
392 */
393 struct __packet pkt_com;
394
395 /*
396 * Option common area (PKT_F_OPT_DATA),
397 * non-NULL if PKT_F_OPT_ALLOC is set.
398 */
399 struct __packet_opt *pkt_com_opt;
400
401 /* TX: enqueue time, RX: receive timestamp */
402 uint64_t pkt_timestamp;
403
404 /* next chain in queue; used while enqueuing to classq or reass */
405 struct __kern_packet *pkt_nextpkt;
406
407 /*
408 * Attached mbuf or pkt.
409 * Used by compat netif driver (PKT_F_MBUF_DATA) or interface
410 * filters (PKT_F_PKT_DATA).
411 */
412 union {
413 struct mbuf *pkt_mbuf;
414 struct __kern_packet *pkt_pkt;
415 };
416 /*
417 * Flow classifier data (PKT_F_FLOW_DATA),
418 * non-NULL if PKT_F_FLOW_ALLOC is set.
419 */
420 struct __flow *pkt_flow; /* classifier info */
421 #define pkt_flow_ipv4_addrs pkt_flow->flow_ipv4_addrs
422 #define pkt_flow_ipv4_src pkt_flow->flow_ipv4_src
423 #define pkt_flow_ipv4_dst pkt_flow->flow_ipv4_dst
424 #define pkt_flow_ipv6_addrs pkt_flow->flow_ipv6_addrs
425 #define pkt_flow_ipv6_src pkt_flow->flow_ipv6_src
426 #define pkt_flow_ipv6_dst pkt_flow->flow_ipv6_dst
427 #define pkt_flow_ip_ver pkt_flow->flow_ip_ver
428 #define pkt_flow_ip_proto pkt_flow->flow_ip_proto
429 #define pkt_flow_ip_hdr pkt_flow->flow_ip_hdr
430 #define pkt_flow_tcp pkt_flow->flow_tcp
431 #define pkt_flow_tcp_src pkt_flow->flow_tcp_src
432 #define pkt_flow_tcp_dst pkt_flow->flow_tcp_dst
433 #define pkt_flow_tcp_seq pkt_flow->flow_tcp_seq
434 #define pkt_flow_tcp_ack pkt_flow->flow_tcp_ack
435 #define pkt_flow_tcp_off pkt_flow->flow_tcp_off
436 #define pkt_flow_tcp_flags pkt_flow->flow_tcp_flags
437 #define pkt_flow_tcp_win pkt_flow->flow_tcp_win
438 #define pkt_flow_tcp_hlen pkt_flow->flow_tcp_hlen
439 #define pkt_flow_tcp_hdr pkt_flow->flow_tcp_hdr
440 #define pkt_flow_tcp_agg_fast pkt_flow->flow_tcp_agg_fast
441 #define pkt_flow_udp pkt_flow->flow_udp
442 #define pkt_flow_udp_src pkt_flow->flow_udp_src
443 #define pkt_flow_udp_dst pkt_flow->flow_udp_dst
444 #define pkt_flow_udp_hlen pkt_flow->flow_udp_hlen
445 #define pkt_flow_udp_hdr pkt_flow->flow_udp_hdr
446 #define pkt_flow_esp_spi pkt_flow->flow_esp_spi
447 #define pkt_transport_protocol pkt_flow->flow_ulp_encap
448 #define pkt_flow_ip_hlen pkt_flow->flow_ip_hlen
449 #define pkt_flow_ulen pkt_flow->flow_ulen
450 #define pkt_flow_ip_frag_id pkt_flow->flow_ip_frag_id
451 #define pkt_flow_ip_is_frag pkt_flow->flow_ip_is_frag
452 #define pkt_flow_ip_is_first_frag pkt_flow->flow_ip_is_first_frag
453 #define pkt_flowsrc_token pkt_flow->flow_src_token
454 #define pkt_flowsrc_id pkt_flow->flow_src_id
455 #define pkt_flowsrc_fidx pkt_flow->flow_src_fidx
456 #define pkt_flowsrc_type pkt_flow->flow_src_type
457 #define pkt_classq_hash pkt_flow->flow_classq_hash
458 #define pkt_classq_flags pkt_flow->flow_classq_flags
459 #define pkt_policy_id pkt_flow->flow_policy_id
460 #define pkt_policy_euuid pkt_flow->flow_policy_euuid
461
462 /*
463 * Transmit completion data (PKT_TX_COMPL_DATA),
464 * non-NULL if PKT_F_TX_COMPL_ALLOC is set.
465 */
466 struct __packet_compl *pkt_tx_compl; /* TX completion info */
467 #define pkt_tx_compl_data pkt_tx_compl->compl_data
468 #define pkt_tx_compl_data64 pkt_tx_compl->compl_data64
469 #define pkt_tx_compl_cb_arg pkt_tx_compl->compl_data._cb_arg
470 #define pkt_tx_compl_cb_data pkt_tx_compl->compl_data._cb_data
471 #define pkt_tx_compl_status pkt_tx_compl->compl_data._tx_status
472 #define pkt_tx_compl_callbacks pkt_tx_compl->compl_callbacks
473 #define pkt_tx_compl_context pkt_tx_compl->compl_context
474
475 void * pkt_priv; /* free to use for every layer */
476
477
478 /*
479 * Kernel specific.
480 *
481 * pkt_{bufs,max} aren't part of the common area, on purpose,
482 * since we selectively update them on internalize/externalize.
483 */
484 const uint16_t pkt_bufs_max; /* maximum size of buflet chain */
485 const uint16_t pkt_bufs_cnt; /* buflet chain size */
486 uint32_t pkt_chain_count; /* number of packets in chain */
487 uint32_t pkt_chain_bytes; /* number of bytes in chain */
488
489 nexus_port_t pkt_nx_port; /* user channel port */
490 /*
491 * gencnt of pkt_nx_port's corresponding vpna. So that we can tell
492 * whether the port in pkt_nx_port has been defuncted or reused.
493 */
494 uint16_t pkt_vpna_gencnt;
495
496 /* Cellular Host Driver generated trace_tag */
497 packet_trace_tag_t pkt_trace_tag;
498 /* index of the qset that the pkt comes from */
499 uint8_t pkt_qset_idx;
500 uint8_t _pad[1];
501 } __attribute((aligned(sizeof(uint64_t))));
502
503
504 /* the size of __user_packet structure for n total buflets */
505 #define _KERN_PACKET_SIZE(n) sizeof(struct __kern_packet)
506
507 #define _PKT_COM_INIT(_p, _pflags) do { \
508 /* save packet flags since it might be wiped out */ \
509 volatile uint64_t __pflags = (_pflags); \
510 /* first wipe it clean */ \
511 _CASSERT(sizeof(struct __packet_com) == 32); \
512 _CASSERT(sizeof(struct __packet) == 32); \
513 sk_zero_32(&(_p)->pkt_com.__pkt_data[0]); \
514 /* then initialize */ \
515 (_p)->pkt_pflags = (__pflags); \
516 (_p)->pkt_svc_class = KPKT_SC_UNSPEC; \
517 } while (0)
518
519 #define _PKT_CTOR(_p, _pflags, _bufcnt, _maxfrags) do { \
520 _PKT_COM_INIT(_p, _pflags); \
521 _CASSERT(sizeof ((_p)->pkt_bufs_max) == sizeof (uint16_t)); \
522 _CASSERT(sizeof ((_p)->pkt_bufs_cnt) == sizeof (uint16_t)); \
523 /* deconst */ \
524 *(uint16_t *)(uintptr_t)&(_p)->pkt_bufs_max = (_maxfrags); \
525 *(uint16_t *)(uintptr_t)&(_p)->pkt_bufs_cnt = (_bufcnt); \
526 } while (0)
527
528 #define KPKT_CLEAR_MBUF_PKT_DATA(_pk) do { \
529 _CASSERT(offsetof(struct __kern_packet, pkt_mbuf) == \
530 offsetof(struct __kern_packet, pkt_pkt)); \
531 (_pk)->pkt_pflags &= ~(PKT_F_MBUF_MASK|PKT_F_PKT_MASK); \
532 /* the following also clears pkt_pkt */ \
533 (_pk)->pkt_mbuf = NULL; \
534 } while (0)
535
536 #define KPKT_CLEAR_MBUF_DATA(_pk) do { \
537 (_pk)->pkt_pflags &= ~PKT_F_MBUF_MASK; \
538 (_pk)->pkt_mbuf = NULL; \
539 } while (0)
540
541 #define KPKT_CLEAR_PKT_DATA(_pk) do { \
542 (_pk)->pkt_pflags &= ~PKT_F_PKT_MASK; \
543 (_pk)->pkt_pkt = NULL; \
544 } while (0)
545
546 #define KPKT_CLEAR_FLOW_INIT(_fl) do { \
547 _CASSERT(sizeof ((_fl)->flow_init_data) == 128); \
548 sk_zero_128(&(_fl)->flow_init_data[0]); \
549 } while (0)
550
551 #define KPKT_CLEAR_FLOW_ALL(_fl) do { \
552 bzero(_fl, sizeof(struct __flow)); \
553 } while (0)
554
555 #define _KPKT_CTOR_PRIV_VARS(_p, _opt, _flow, _txcomp) do { \
556 (_p)->pkt_com_opt = (_opt); \
557 (_p)->pkt_flow = (_flow); \
558 (_p)->pkt_tx_compl = (_txcomp); \
559 } while (0)
560
561 #define _KPKT_INIT_FPD_VARS(_p)
562
563 #define _KPKT_INIT_PRIV_VARS(_p) do { \
564 struct __flow *__fl = (_p)->pkt_flow; \
565 (_p)->pkt_timestamp = 0; \
566 (_p)->pkt_nextpkt = NULL; \
567 (_p)->pkt_priv = NULL; \
568 _KPKT_INIT_FPD_VARS(_p); \
569 KPKT_CLEAR_MBUF_PKT_DATA(_p); \
570 if (__probable(__fl != NULL)) { \
571 KPKT_CLEAR_FLOW_INIT(__fl); \
572 } \
573 (_p)->pkt_chain_count = (_p)->pkt_chain_bytes = 0; \
574 (_p)->pkt_nx_port = NEXUS_PORT_ANY; \
575 (_p)->pkt_vpna_gencnt = 0; \
576 (_p)->pkt_trace_tag = 0; \
577 (_p)->pkt_qset_idx = 0; \
578 } while (0)
579
580 #define KPKT_CTOR(_pk, _pflags, _opt, _flow, _txcomp, _midx, _pu, _pp, \
581 _bufcnt, _maxfrags, _qflags) do { \
582 ASSERT((uintptr_t)(_pk) != (uintptr_t)(_pu)); \
583 /* ASSERT((_pu) != NULL || PP_KERNEL_ONLY(_pp)); */ \
584 /* kernel (and user) quantum */ \
585 KQUM_CTOR(&(_pk)->pkt_qum, _midx, \
586 (((_pu) == NULL) ? NULL : &(_pu)->pkt_qum), _pp, _qflags); \
587 /* kernel packet variant */ \
588 _PKT_CTOR(_pk, _pflags, _bufcnt, _maxfrags); \
589 _KPKT_CTOR_PRIV_VARS(_pk, _opt, _flow, _txcomp); \
590 /* no need to construct user variant as it is done in externalize */ \
591 } while (0)
592
593 #define KPKT_INIT(_pk, _flags) do { \
594 KQUM_INIT(&(_pk)->pkt_qum, _flags); \
595 _PKT_COM_INIT(_pk, (_pk)->pkt_pflags); \
596 _KPKT_INIT_PRIV_VARS(_pk); \
597 /* no need to initialize user variant as it is done in externalize */ \
598 } while (0)
599
600 #define _KPKT_INIT_TX_COMPL_DATA(_p) do { \
601 if (((_p)->pkt_pflags & PKT_F_TX_COMPL_DATA) == 0) { \
602 ASSERT((_p)->pkt_pflags & PKT_F_TX_COMPL_ALLOC); \
603 (_p)->pkt_pflags |= PKT_F_TX_COMPL_DATA; \
604 _CASSERT(sizeof((_p)->pkt_tx_compl_data64) == 24); \
605 /* 32-bit compl_data should be in the union */ \
606 _CASSERT(sizeof((_p)->pkt_tx_compl_data) <= 24); \
607 (_p)->pkt_tx_compl_data64[0] = 0; \
608 (_p)->pkt_tx_compl_data64[1] = 0; \
609 (_p)->pkt_tx_compl_data64[2] = 0; \
610 } \
611 } while (0)
612
613 /*
614 * Copy optional meta data.
615 * Both source and destination must be a kernel packet.
616 */
617 #define _PKT_COPY_OPT_DATA(_skp, _dkp) do { \
618 if (__improbable(((_skp)->pkt_pflags & PKT_F_OPT_DATA) != 0)) { \
619 _CASSERT(sizeof(struct __packet_opt) == 32); \
620 ASSERT((_skp)->pkt_pflags & PKT_F_OPT_ALLOC); \
621 sk_copy64_32((uint64_t *)(void *)(_skp)->pkt_com_opt, \
622 (uint64_t *)(void *)(_dkp)->pkt_com_opt); \
623 } \
624 } while (0)
625
626 /*
627 * _PKT_COPY only copies the user metadata portion of the packet;
628 * at the moment this is everything from the beginning down to __p_flags,
629 * but no more. It additionally copies only QUM_F_COPY_MASK bits from
630 * the source __p_flags to the destination's.
631 *
632 * NOTE: this needs to be adjusted if more user-mutable field is added
633 * after __p_flags.
634 */
635 #define _PKT_COPY(_skp, _dkp) do { \
636 _CASSERT(sizeof(struct __packet) == 32); \
637 _CASSERT(sizeof(struct __packet_com) == 32); \
638 _CASSERT(offsetof(struct __packet, __p_flags) == 24); \
639 /* copy __packet excluding pkt_pflags */ \
640 sk_copy64_24((uint64_t *)(void *)&(_skp)->pkt_com, \
641 (uint64_t *)(void *)&(_dkp)->pkt_com); \
642 /* copy relevant pkt_pflags bits */ \
643 (_dkp)->pkt_pflags = ((_skp)->pkt_pflags & PKT_F_COPY_MASK); \
644 /* copy __packet_opt if applicable */ \
645 _PKT_COPY_OPT_DATA((_skp), (_dkp)); \
646 } while (0)
647
648
649 /*
650 * Copy Transmit completion data.
651 */
652 #define _PKT_COPY_TX_PORT_DATA(_skp, _dkp) do { \
653 (_dkp)->pkt_nx_port = (_skp)->pkt_nx_port; \
654 (_dkp)->pkt_vpna_gencnt = (_skp)->pkt_vpna_gencnt; \
655 (_dkp)->pkt_pflags |= ((_skp)->pkt_pflags & PKT_F_TX_PORT_DATA);\
656 } while (0)
657
658 /*
659 * _PKT_INTERNALIZE internalizes a portion of the packet that includes
660 * user visible fields without overwriting the portion that's private to
661 * the kernel.
662 *
663 * NOTE: this needs to be adjusted if more user-mutable data is added
664 * after __p_flags. This macro is used only during internalize.
665 */
666 #define _PKT_INTERNALIZE(_up, _kp) do { \
667 volatile uint64_t _kf = ((_kp)->pkt_pflags & ~PKT_F_USER_MASK); \
668 _CASSERT(sizeof(struct __packet) == 32); \
669 _CASSERT(sizeof(struct __packet_com) == 32); \
670 _CASSERT(offsetof(struct __packet, __p_flags) == 24); \
671 /* copy __packet excluding pkt_pflags */ \
672 sk_copy64_24((uint64_t *)(void *)&(_up)->pkt_com, \
673 (uint64_t *)(void *)&(_kp)->pkt_com); \
674 /* copy pkt_pflags and restore kernel bits */ \
675 (_kp)->pkt_pflags = ((_up)->pkt_pflags & PKT_F_USER_MASK) | _kf;\
676 /* copy (internalize) __packet_opt if applicable */ \
677 if (__improbable(((_kp)->pkt_pflags & PKT_F_OPT_DATA) != 0)) { \
678 _CASSERT(sizeof(struct __packet_opt) == 32); \
679 ASSERT((_kp)->pkt_pflags & PKT_F_OPT_ALLOC); \
680 sk_copy64_32((uint64_t *)(void *)&(_up)->pkt_com_opt, \
681 (uint64_t *)(void *)(_kp)->pkt_com_opt); \
682 } \
683 } while (0)
684
685 /*
686 * _PKT_EXTERNALIZE externalizes a portion of the packet that's user
687 * visible without including fields that's private to the kernel; at the
688 * moment this is everything from the beginning down to __p_flags,
689 * but no more.
690 *
691 * NOTE: this needs to be adjusted if more user-mutable data is added
692 * after __p_flags. This macro is used only during externalize.
693 */
694 #define _PKT_EXTERNALIZE(_kp, _up) do { \
695 _CASSERT(sizeof(struct __packet) == 32); \
696 _CASSERT(sizeof(struct __packet_com) == 32); \
697 _CASSERT(offsetof(struct __packet, __p_flags) == 24); \
698 /* copy __packet excluding pkt_pflags */ \
699 sk_copy64_24((uint64_t *)(void *)&(_kp)->pkt_com, \
700 (uint64_t *)(void *)&(_up)->pkt_com); \
701 /* copy pkt_pflags excluding kernel bits */ \
702 (_up)->pkt_pflags = ((_kp)->pkt_pflags & PKT_F_USER_MASK); \
703 /* copy (externalize) __packet_opt if applicable */ \
704 if (__improbable(((_kp)->pkt_pflags & PKT_F_OPT_DATA) != 0)) { \
705 _CASSERT(sizeof(struct __packet_opt) == 32); \
706 ASSERT((_kp)->pkt_pflags & PKT_F_OPT_ALLOC); \
707 sk_copy64_32((uint64_t *)(void *)(_kp)->pkt_com_opt, \
708 (uint64_t *)(void *)&(_up)->pkt_com_opt); \
709 } \
710 } while (0)
711
712 #define SK_PTR_ADDR_KQUM(_ph) ((struct __kern_quantum *)SK_PTR_ADDR(_ph))
713 #define SK_PTR_ADDR_KPKT(_ph) ((struct __kern_packet *)SK_PTR_ADDR(_ph))
714 #define SK_PTR_KPKT(_pa) ((struct __kern_packet *)(void *)(_pa))
715 #define SK_PKT2PH(_pkt) \
716 (SK_PTR_ENCODE((_pkt), METADATA_TYPE((_pkt)), METADATA_SUBTYPE((_pkt))))
717
718 /*
719 * Set the length of the data to various places: __user_slot_desc,
720 * __kern_quantum, and for a packet, the buflet.
721 * !!! This should be used only for dropping the packet as the macro
722 * is not functionally correct.
723 *
724 * TODO: [email protected] -- maybe finalize here as well?
725 */
726 #define METADATA_SET_LEN(_md, _len, _doff) do { \
727 struct __kern_quantum *_q = \
728 (struct __kern_quantum *)(void *)(_md); \
729 _q->qum_len = (_len); \
730 switch (METADATA_TYPE(_q)) { \
731 case NEXUS_META_TYPE_PACKET: { \
732 struct __kern_packet *_p = \
733 (struct __kern_packet *)(void *)(_md); \
734 struct __kern_buflet *_kbft; \
735 PKT_GET_FIRST_BUFLET(_p, _p->pkt_bufs_cnt, _kbft); \
736 _kbft->buf_dlen = (_len); \
737 _kbft->buf_doff = (_doff); \
738 break; \
739 } \
740 default: \
741 ASSERT(METADATA_TYPE(_q) == NEXUS_META_TYPE_QUANTUM); \
742 _q->qum_buf[0].buf_dlen = (_len); \
743 _q->qum_buf[0].buf_doff = (_doff); \
744 break; \
745 } \
746 } while (0)
747
748 #define METADATA_ADJUST_LEN(_md, _len, _doff) do { \
749 struct __kern_quantum *_q = \
750 (struct __kern_quantum *)(void *)(_md); \
751 switch (METADATA_TYPE(_q)) { \
752 case NEXUS_META_TYPE_PACKET: { \
753 struct __kern_packet *_p = \
754 (struct __kern_packet *)(void *)(_md); \
755 struct __kern_buflet *_kbft; \
756 PKT_GET_FIRST_BUFLET(_p, _p->pkt_bufs_cnt, _kbft); \
757 _kbft->buf_dlen += (_len); \
758 _kbft->buf_doff = (_doff); \
759 break; \
760 } \
761 default: \
762 ASSERT(METADATA_TYPE(_q) == NEXUS_META_TYPE_QUANTUM); \
763 _q->qum_buf[0].buf_dlen += (_len); \
764 _q->qum_buf[0].buf_doff = (_doff); \
765 break; \
766 } \
767 } while (0)
768
769 __attribute__((always_inline))
770 static inline kern_packet_t
SD_GET_TAGGED_METADATA(const struct __kern_slot_desc * ksd)771 SD_GET_TAGGED_METADATA(const struct __kern_slot_desc *ksd)
772 {
773 return __improbable(ksd->sd_md == NULL) ? 0 :
774 SK_PTR_ENCODE(ksd->sd_md, METADATA_TYPE(ksd->sd_qum),
775 METADATA_SUBTYPE(ksd->sd_qum));
776 }
777
778 __attribute__((always_inline))
779 static inline errno_t
KR_SLOT_ATTACH_METADATA(const kern_channel_ring_t kring,struct __kern_slot_desc * ksd,struct __kern_quantum * kqum)780 KR_SLOT_ATTACH_METADATA(const kern_channel_ring_t kring,
781 struct __kern_slot_desc *ksd, struct __kern_quantum *kqum)
782 {
783 obj_idx_t idx = KR_SLOT_INDEX(kring,
784 (struct __slot_desc *)(void *)ksd);
785
786 /* Ensure this is only done by the thread doing a sync syscall */
787 ASSERT(sk_is_sync_protected());
788 ASSERT(kqum->qum_pp == kring->ckr_pp);
789 ASSERT(kqum->qum_ksd == NULL);
790 /*
791 * Packets being attached to a slot should always be internalized.
792 * Internalized packet should be in finalized or dropped state.
793 */
794 ASSERT(kqum->qum_qflags & QUM_F_INTERNALIZED);
795 ASSERT(((kqum->qum_qflags & QUM_F_FINALIZED) != 0) ^
796 ((kqum->qum_qflags & QUM_F_DROPPED) != 0));
797
798 kqum->qum_ksd = ksd;
799
800 KSD_ATTACH_METADATA(ksd, kqum);
801 if (!KR_KERNEL_ONLY(kring)) {
802 USD_ATTACH_METADATA(KR_USD(kring, idx), METADATA_IDX(kqum));
803 }
804
805 return 0;
806 }
807
808 __attribute__((always_inline))
809 static inline struct __kern_quantum *
KR_SLOT_DETACH_METADATA(const kern_channel_ring_t kring,struct __kern_slot_desc * ksd)810 KR_SLOT_DETACH_METADATA(const kern_channel_ring_t kring,
811 struct __kern_slot_desc *ksd)
812 {
813 struct __kern_quantum *kqum = ksd->sd_qum;
814 obj_idx_t idx = KR_SLOT_INDEX(kring,
815 (struct __slot_desc *)(void *)ksd);
816
817 /* Ensure this is only done by the thread doing a sync syscall */
818 ASSERT(sk_is_sync_protected());
819 ASSERT(KSD_VALID_METADATA(ksd));
820 ASSERT(kqum->qum_ksd == ksd);
821 ASSERT(kqum->qum_pp == kring->ckr_pp);
822 /*
823 * Packets being attached to a slot would always be internalized.
824 * We also detach externalized packets on an rx ring on behalf
825 * of the user space if the channel is not in user packet pool mode.
826 * Externalized packet should be in finalized or dropped state.
827 */
828 ASSERT((kqum->qum_qflags & (QUM_F_INTERNALIZED)) ||
829 ((((kqum->qum_qflags & QUM_F_FINALIZED) != 0) ^
830 ((kqum->qum_qflags & QUM_F_DROPPED) != 0))));
831
832 /* detaching requires the packet to be finalized later */
833 kqum->qum_qflags &= ~QUM_F_FINALIZED;
834 kqum->qum_ksd = NULL;
835
836 KSD_DETACH_METADATA(ksd);
837 if (!KR_KERNEL_ONLY(kring)) {
838 USD_DETACH_METADATA(KR_USD(kring, idx));
839 }
840
841 return kqum;
842 }
843
844 __attribute__((always_inline))
845 static inline errno_t
KR_SLOT_ATTACH_BUF_METADATA(const kern_channel_ring_t kring,struct __kern_slot_desc * ksd,struct __kern_buflet * kbuf)846 KR_SLOT_ATTACH_BUF_METADATA(const kern_channel_ring_t kring,
847 struct __kern_slot_desc *ksd, struct __kern_buflet *kbuf)
848 {
849 obj_idx_t idx = KR_SLOT_INDEX(kring,
850 (struct __slot_desc *)(void *)ksd);
851
852 /* Ensure this is only done by the thread doing a sync syscall */
853 ASSERT(sk_is_sync_protected());
854
855 KSD_ATTACH_METADATA(ksd, kbuf);
856 /*
857 * buflet is attached only to the user packet pool alloc ring.
858 */
859 ASSERT(!KR_KERNEL_ONLY(kring));
860 ASSERT(kring->ckr_tx == CR_KIND_ALLOC);
861 USD_ATTACH_METADATA(KR_USD(kring, idx), kbuf->buf_bft_idx_reg);
862 return 0;
863 }
864
865 #if (DEVELOPMENT || DEBUG)
866 SYSCTL_DECL(_kern_skywalk_packet);
867 extern int pkt_trailers;
868 #endif /* !DEVELOPMENT && !DEBUG */
869
870 typedef void (pkt_copy_from_pkt_t)(const enum txrx, kern_packet_t,
871 const uint16_t, kern_packet_t, const uint16_t, const uint32_t,
872 const boolean_t, const uint16_t, const uint16_t, const boolean_t);
873
874 typedef void (pkt_copy_from_mbuf_t)(const enum txrx, kern_packet_t,
875 const uint16_t, struct mbuf *, const uint16_t, const uint32_t,
876 const boolean_t, const uint16_t);
877
878 typedef void (pkt_copy_to_mbuf_t)(const enum txrx, kern_packet_t,
879 const uint16_t, struct mbuf *, const uint16_t, const uint32_t,
880 const boolean_t, const uint16_t);
881
882 __BEGIN_DECLS
883 extern void pkt_subtype_assert_fail(const kern_packet_t, uint64_t, uint64_t);
884 extern void pkt_type_assert_fail(const kern_packet_t, uint64_t);
885
886 extern pkt_copy_from_pkt_t pkt_copy_from_pkt;
887 extern pkt_copy_from_pkt_t pkt_copy_multi_buflet_from_pkt;
888 extern pkt_copy_from_mbuf_t pkt_copy_from_mbuf;
889 extern pkt_copy_from_mbuf_t pkt_copy_multi_buflet_from_mbuf;
890 extern pkt_copy_to_mbuf_t pkt_copy_to_mbuf;
891 extern pkt_copy_to_mbuf_t pkt_copy_multi_buflet_to_mbuf;
892
893 extern void pkt_copypkt_sum(kern_packet_t, uint16_t, kern_packet_t,
894 uint16_t, uint16_t, uint32_t *, boolean_t);
895 extern uint32_t
896 pkt_copyaddr_sum(kern_packet_t sph, uint16_t soff, uint8_t *dbaddr,
897 uint32_t len, boolean_t do_csum, uint32_t initial_sum, boolean_t *odd_start);
898 extern uint32_t pkt_sum(kern_packet_t, uint16_t, uint16_t);
899 extern uint32_t pkt_mcopypkt_sum(mbuf_t, int, kern_packet_t, uint16_t,
900 uint16_t, boolean_t);
901 extern uint32_t
902 m_copydata_sum(struct mbuf *m, int off, int len, void *vp, uint32_t initial_sum,
903 boolean_t *odd_start);
904 extern void pkt_copy(void *src, void *dst, size_t len);
905
906 #if (DEVELOPMENT || DEBUG)
907 extern uint32_t pkt_add_trailers(kern_packet_t, const uint32_t, const uint16_t);
908 extern uint32_t pkt_add_trailers_mbuf(struct mbuf *, const uint16_t);
909 #endif /* !DEVELOPMENT && !DEBUG */
910 __END_DECLS
911 #endif /* BSD_KERNEL_PRIVATE */
912 #endif /* !_SKYWALK_PACKET_PACKETVAR_H_ */
913