1 /*
2 * Copyright (c) 2012-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/locks.h>
30 #include <kern/zalloc.h>
31
32 #include <sys/types.h>
33 #include <sys/kernel_types.h>
34 #include <sys/kauth.h>
35 #include <sys/socket.h>
36 #include <sys/socketvar.h>
37 #include <sys/sockio.h>
38 #include <sys/sysctl.h>
39 #include <sys/proc.h>
40
41 #include <net/if.h>
42 #include <net/if_var.h>
43 #include <net/if_types.h>
44 #include <net/bpf.h>
45 #include <net/net_osdep.h>
46 #include <net/pktap.h>
47
48 #include <netinet/in_pcb.h>
49 #include <netinet/tcp.h>
50 #include <netinet/tcp_var.h>
51 #define _IP_VHL
52 #include <netinet/ip.h>
53 #include <netinet/ip_var.h>
54 #include <netinet/udp.h>
55 #include <netinet/udp_var.h>
56
57 #include <netinet/ip6.h>
58 #include <netinet6/in6_pcb.h>
59
60 #include <libkern/OSAtomic.h>
61
62 #include <kern/debug.h>
63
64 #include <os/log.h>
65
66 #include <sys/mcache.h>
67
68 #include <string.h>
69 #include <stdbool.h>
70
71
72 struct kern_pktap_filter {
73 uint32_t filter_op;
74 uint32_t filter_param;
75 union {
76 uint32_t _filter_if_type;
77 char _filter_if_name[PKTAP_IFXNAMESIZE];
78 } param_;
79 size_t filter_ifname_len;
80 bool filter_ifname_prefix_match;
81 };
82
83 extern struct inpcbinfo ripcbinfo;
84
85 struct pktap_softc {
86 LIST_ENTRY(pktap_softc) pktp_link;
87 uint32_t pktp_unit;
88 uint32_t pktp_dlt_raw_count;
89 uint32_t pktp_dlt_pkttap_count;
90 struct ifnet *pktp_ifp;
91 struct kern_pktap_filter pktp_filters[PKTAP_MAX_FILTERS];
92 };
93
94 #ifndef PKTAP_DEBUG
95 #define PKTAP_DEBUG 0
96 #endif /* PKTAP_DEBUG */
97
98 #define PKTAP_FILTER_OK 0 /* Packet passes filter checks */
99 #define PKTAP_FILTER_SKIP 1 /* Do not tap this packet */
100
101 static int pktap_inited = 0;
102
103 SYSCTL_DECL(_net_link);
104 SYSCTL_NODE(_net_link, IFT_PKTAP, pktap,
105 CTLFLAG_RW | CTLFLAG_LOCKED, 0, "pktap virtual interface");
106
107 uint32_t pktap_total_tap_count = 0;
108 SYSCTL_UINT(_net_link_pktap, OID_AUTO, total_tap_count,
109 CTLFLAG_RD | CTLFLAG_LOCKED, &pktap_total_tap_count, 0, "");
110
111 static u_int64_t pktap_count_unknown_if_type = 0;
112 SYSCTL_QUAD(_net_link_pktap, OID_AUTO, count_unknown_if_type,
113 CTLFLAG_RD | CTLFLAG_LOCKED, &pktap_count_unknown_if_type, "");
114
115 static int pktap_log = 0;
116 SYSCTL_INT(_net_link_pktap, OID_AUTO, log,
117 CTLFLAG_RW | CTLFLAG_LOCKED, &pktap_log, 0, "");
118
119 #define PKTAP_LOG(mask, fmt, ...) \
120 do { \
121 if (__improbable(pktap_log & mask)) \
122 os_log(OS_LOG_DEFAULT, "%s:%d " fmt, __FUNCTION__, __LINE__, ##__VA_ARGS__); \
123 } while (false)
124
125 #define PKTP_LOG_FUNC 0x01
126 #define PKTP_LOG_FILTER 0x02
127 #define PKTP_LOG_INPUT 0x04
128 #define PKTP_LOG_OUTPUT 0x08
129 #define PKTP_LOG_ERROR 0x10
130 #define PKTP_LOG_NOPCB 0x20
131
132 /*
133 * pktap_lck_rw protects the global list of pktap interfaces
134 */
135 static LCK_GRP_DECLARE(pktap_lck_grp, "pktap");
136 #if PKTAP_DEBUG
137 static LCK_ATTR_DECLARE(pktap_lck_attr, LCK_ATTR_DEBUG, 0);
138 #else
139 static LCK_ATTR_DECLARE(pktap_lck_attr, 0, 0);
140 #endif
141 static LCK_RW_DECLARE_ATTR(pktap_lck_rw, &pktap_lck_grp, &pktap_lck_attr);
142
143
144 static LIST_HEAD(pktap_list, pktap_softc) pktap_list =
145 LIST_HEAD_INITIALIZER(pktap_list);
146
147 int pktap_clone_create(struct if_clone *, u_int32_t, void *);
148 int pktap_clone_destroy(struct ifnet *);
149
150 #define PKTAP_MAXUNIT IF_MAXUNIT
151 #define PKTAP_ZONE_MAX_ELEM MIN(IFNETS_MAX, PKTAP_MAXUNIT)
152
153 static struct if_clone pktap_cloner =
154 IF_CLONE_INITIALIZER(PKTAP_IFNAME,
155 pktap_clone_create,
156 pktap_clone_destroy,
157 0,
158 PKTAP_MAXUNIT);
159
160 errno_t pktap_if_output(ifnet_t, mbuf_t);
161 errno_t pktap_demux(ifnet_t, mbuf_t, char *, protocol_family_t *);
162 errno_t pktap_add_proto(ifnet_t, protocol_family_t,
163 const struct ifnet_demux_desc *, u_int32_t);
164 errno_t pktap_del_proto(ifnet_t, protocol_family_t);
165 errno_t pktap_getdrvspec(ifnet_t, struct ifdrv64 *);
166 errno_t pktap_setdrvspec(ifnet_t, struct ifdrv64 *);
167 errno_t pktap_ioctl(ifnet_t, unsigned long cmd, void *__sized_by(IOCPARM_LEN(cmd)));
168 void pktap_detach(ifnet_t);
169 int pktap_filter_evaluate(struct pktap_softc *, struct ifnet *);
170 void pktap_bpf_tap(struct ifnet *, protocol_family_t, struct mbuf *,
171 u_int32_t, u_int32_t, int);
172 errno_t pktap_tap_callback(ifnet_t, u_int32_t, bpf_tap_mode);
173
174 static void
pktap_hexdump(int mask,void * __sized_by (len)addr,size_t len)175 pktap_hexdump(int mask, void *__sized_by(len) addr, size_t len)
176 {
177 unsigned char *buf = addr;
178 size_t i;
179
180 if (!(pktap_log & mask)) {
181 return;
182 }
183
184 for (i = 0; i < len; i++) {
185 unsigned char h = (buf[i] & 0xf0) >> 4;
186 unsigned char l = buf[i] & 0x0f;
187
188 if (i != 0) {
189 if (i % 32 == 0) {
190 printf("\n");
191 } else if (i % 4 == 0) {
192 printf(" ");
193 }
194 }
195 printf("%c%c",
196 h < 10 ? h + '0' : h - 10 + 'a',
197 l < 10 ? l + '0' : l - 10 + 'a');
198 }
199 if (i % 32 != 0) {
200 printf("\n");
201 }
202 }
203
204 #define _CASSERT_OFFFSETOF_FIELD(s1, s2, f) \
205 _CASSERT(offsetof(struct s1, f) == offsetof(struct s2, f))
206
207 __private_extern__ void
pktap_init(void)208 pktap_init(void)
209 {
210 int error = 0;
211
212 _CASSERT_OFFFSETOF_FIELD(pktap_header, pktap_v2_hdr, pth_flags);
213
214 /* Make sure we're called only once */
215 VERIFY(pktap_inited == 0);
216
217 pktap_inited = 1;
218
219 LIST_INIT(&pktap_list);
220
221 error = if_clone_attach(&pktap_cloner);
222 if (error != 0) {
223 panic("%s: if_clone_attach() failed, error %d",
224 __func__, error);
225 }
226 }
227
228 __private_extern__ int
pktap_clone_create(struct if_clone * ifc,u_int32_t unit,__unused void * params)229 pktap_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params)
230 {
231 int error = 0;
232 struct pktap_softc *__single pktap = NULL;
233 struct ifnet_init_eparams if_init;
234
235 PKTAP_LOG(PKTP_LOG_FUNC, "unit %u\n", unit);
236
237 pktap = kalloc_type(struct pktap_softc, Z_WAITOK_ZERO_NOFAIL);
238 pktap->pktp_unit = unit;
239
240 /*
241 * By default accept packet from physical interfaces
242 */
243 pktap->pktp_filters[0].filter_op = PKTAP_FILTER_OP_PASS;
244 pktap->pktp_filters[0].filter_param = PKTAP_FILTER_PARAM_IF_TYPE;
245 pktap->pktp_filters[0].filter_param_if_type = IFT_ETHER;
246
247 #if !XNU_TARGET_OS_OSX
248 pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS;
249 pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE;
250 pktap->pktp_filters[1].filter_param_if_type = IFT_CELLULAR;
251 #else /* XNU_TARGET_OS_OSX */
252 pktap->pktp_filters[1].filter_op = PKTAP_FILTER_OP_PASS;
253 pktap->pktp_filters[1].filter_param = PKTAP_FILTER_PARAM_IF_TYPE;
254 pktap->pktp_filters[1].filter_param_if_type = IFT_IEEE1394;
255 #endif /* XNU_TARGET_OS_OSX */
256
257 pktap->pktp_filters[2].filter_op = PKTAP_FILTER_OP_PASS;
258 pktap->pktp_filters[2].filter_param = PKTAP_FILTER_PARAM_IF_TYPE;
259 pktap->pktp_filters[2].filter_param_if_type = IFT_OTHER;
260
261 /*
262 * We do not use a set_bpf_tap() function as we rather rely on the more
263 * accurate callback passed to bpf_attach()
264 */
265 bzero(&if_init, sizeof(if_init));
266 if_init.ver = IFNET_INIT_CURRENT_VERSION;
267 if_init.len = sizeof(if_init);
268 if_init.flags = IFNET_INIT_LEGACY;
269 if_init.name = __unsafe_null_terminated_from_indexable(ifc->ifc_name);
270 if_init.unit = unit;
271 if_init.type = IFT_PKTAP;
272 if_init.family = IFNET_FAMILY_LOOPBACK;
273 if_init.output = pktap_if_output;
274 if_init.demux = pktap_demux;
275 if_init.add_proto = pktap_add_proto;
276 if_init.del_proto = pktap_del_proto;
277 if_init.softc = pktap;
278 if_init.ioctl = pktap_ioctl;
279 if_init.detach = pktap_detach;
280
281 error = ifnet_allocate_extended(&if_init, &pktap->pktp_ifp);
282 if (error != 0) {
283 printf("%s: ifnet_allocate failed, error %d\n",
284 __func__, error);
285 goto done;
286 }
287
288 ifnet_set_flags(pktap->pktp_ifp, IFF_UP, IFF_UP);
289
290 error = ifnet_attach(pktap->pktp_ifp, NULL);
291 if (error != 0) {
292 printf("%s: ifnet_attach failed - error %d\n", __func__, error);
293 ifnet_release(pktap->pktp_ifp);
294 goto done;
295 }
296
297 /* Attach DLT_PKTAP as the default DLT */
298 bpf_attach(pktap->pktp_ifp, DLT_PKTAP, sizeof(struct pktap_header),
299 NULL, pktap_tap_callback);
300 bpf_attach(pktap->pktp_ifp, DLT_RAW, 0, NULL, pktap_tap_callback);
301
302 /* Take a reference and add to the global list */
303 ifnet_reference(pktap->pktp_ifp);
304 lck_rw_lock_exclusive(&pktap_lck_rw);
305 LIST_INSERT_HEAD(&pktap_list, pktap, pktp_link);
306 lck_rw_done(&pktap_lck_rw);
307 done:
308 if (error != 0 && pktap != NULL) {
309 kfree_type(struct pktap_softc, pktap);
310 }
311 return error;
312 }
313
314 __private_extern__ int
pktap_clone_destroy(struct ifnet * ifp)315 pktap_clone_destroy(struct ifnet *ifp)
316 {
317 int error = 0;
318
319 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
320
321 (void) ifnet_detach(ifp);
322
323 return error;
324 }
325
326 /*
327 * This function is called whenever a DLT is set on the interface:
328 * - When interface is attached to a BPF device via BIOCSETIF for the
329 * default DLT
330 * - Whenever a new DLT is selected via BIOCSDLT
331 * - When the interface is detached from a BPF device (direction is zero)
332 */
333 __private_extern__ errno_t
pktap_tap_callback(ifnet_t ifp,u_int32_t dlt,bpf_tap_mode direction)334 pktap_tap_callback(ifnet_t ifp, u_int32_t dlt, bpf_tap_mode direction)
335 {
336 struct pktap_softc *__single pktap;
337
338 pktap = ifp->if_softc;
339 if (pktap == NULL) {
340 printf("%s: if_softc is NULL for ifp %s\n", __func__,
341 ifp->if_xname);
342 goto done;
343 }
344 switch (dlt) {
345 case DLT_RAW:
346 if (direction == 0) {
347 if (pktap->pktp_dlt_raw_count > 0) {
348 pktap->pktp_dlt_raw_count--;
349 OSAddAtomic(-1, &pktap_total_tap_count);
350 }
351 } else {
352 pktap->pktp_dlt_raw_count++;
353 OSAddAtomic(1, &pktap_total_tap_count);
354 }
355 break;
356 case DLT_PKTAP:
357 if (direction == 0) {
358 if (pktap->pktp_dlt_pkttap_count > 0) {
359 pktap->pktp_dlt_pkttap_count--;
360 OSAddAtomic(-1, &pktap_total_tap_count);
361 }
362 } else {
363 pktap->pktp_dlt_pkttap_count++;
364 OSAddAtomic(1, &pktap_total_tap_count);
365 }
366 break;
367 }
368 done:
369 /*
370 * Attachements count must be positive and we're in trouble
371 * if we have more that 2**31 attachements
372 */
373 VERIFY(pktap_total_tap_count >= 0);
374
375 return 0;
376 }
377
378 __private_extern__ errno_t
pktap_if_output(ifnet_t ifp,mbuf_t m)379 pktap_if_output(ifnet_t ifp, mbuf_t m)
380 {
381 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
382 mbuf_freem(m);
383 return ENOTSUP;
384 }
385
386 __private_extern__ errno_t
pktap_demux(ifnet_t ifp,__unused mbuf_t m,__unused char * header,__unused protocol_family_t * ppf)387 pktap_demux(ifnet_t ifp, __unused mbuf_t m, __unused char *header,
388 __unused protocol_family_t *ppf)
389 {
390 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
391 return ENOTSUP;
392 }
393
394 __private_extern__ errno_t
pktap_add_proto(__unused ifnet_t ifp,protocol_family_t pf,__unused const struct ifnet_demux_desc * dmx,__unused u_int32_t cnt)395 pktap_add_proto(__unused ifnet_t ifp, protocol_family_t pf,
396 __unused const struct ifnet_demux_desc *dmx, __unused u_int32_t cnt)
397 {
398 PKTAP_LOG(PKTP_LOG_FUNC, "%s pf %u\n", ifp->if_xname, pf);
399 return 0;
400 }
401
402 __private_extern__ errno_t
pktap_del_proto(__unused ifnet_t ifp,__unused protocol_family_t pf)403 pktap_del_proto(__unused ifnet_t ifp, __unused protocol_family_t pf)
404 {
405 PKTAP_LOG(PKTP_LOG_FUNC, "%s pf %u\n", ifp->if_xname, pf);
406 return 0;
407 }
408
409 __private_extern__ errno_t
pktap_getdrvspec(ifnet_t ifp,struct ifdrv64 * ifd)410 pktap_getdrvspec(ifnet_t ifp, struct ifdrv64 *ifd)
411 {
412 errno_t error = 0;
413 struct pktap_softc *__single pktap;
414 int i;
415
416 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
417
418 pktap = ifp->if_softc;
419 if (pktap == NULL) {
420 error = ENOENT;
421 printf("%s: pktap NULL - error %d\n", __func__, error);
422 goto done;
423 }
424
425 switch (ifd->ifd_cmd) {
426 case PKTP_CMD_FILTER_GET: {
427 struct pktap_filter x_filters[PKTAP_MAX_FILTERS];
428
429 bzero(&x_filters, sizeof(x_filters));
430
431 if (ifd->ifd_len < PKTAP_MAX_FILTERS * sizeof(struct pktap_filter)) {
432 printf("%s: PKTP_CMD_FILTER_GET ifd_len %llu too small - error %d\n",
433 __func__, ifd->ifd_len, error);
434 error = EINVAL;
435 break;
436 }
437 for (i = 0; i < PKTAP_MAX_FILTERS; i++) {
438 struct kern_pktap_filter *pktap_filter = pktap->pktp_filters + i;
439 struct pktap_filter *x_filter = x_filters + i;
440
441 x_filter->filter_op = pktap_filter->filter_op;
442 x_filter->filter_param = pktap_filter->filter_param;
443
444 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_TYPE) {
445 x_filter->filter_param_if_type = pktap_filter->filter_param_if_type;
446 } else if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_NAME) {
447 strbufcpy(x_filter->filter_param_if_name,
448 pktap_filter->filter_param_if_name);
449 }
450 }
451 error = copyout(x_filters, CAST_USER_ADDR_T(ifd->ifd_data),
452 PKTAP_MAX_FILTERS * sizeof(struct pktap_filter));
453 if (error) {
454 printf("%s: PKTP_CMD_FILTER_GET copyout - error %d\n", __func__, error);
455 goto done;
456 }
457 break;
458 }
459 case PKTP_CMD_TAP_COUNT: {
460 uint32_t tap_count = pktap->pktp_dlt_raw_count + pktap->pktp_dlt_pkttap_count;
461
462 if (ifd->ifd_len < sizeof(tap_count)) {
463 printf("%s: PKTP_CMD_TAP_COUNT ifd_len %llu too small - error %d\n",
464 __func__, ifd->ifd_len, error);
465 error = EINVAL;
466 break;
467 }
468 error = copyout(&tap_count, CAST_USER_ADDR_T(ifd->ifd_data), sizeof(tap_count));
469 if (error) {
470 printf("%s: PKTP_CMD_TAP_COUNT copyout - error %d\n", __func__, error);
471 goto done;
472 }
473 break;
474 }
475 default:
476 error = EINVAL;
477 break;
478 }
479
480 done:
481 return error;
482 }
483
484 __private_extern__ errno_t
pktap_setdrvspec(ifnet_t ifp,struct ifdrv64 * ifd)485 pktap_setdrvspec(ifnet_t ifp, struct ifdrv64 *ifd)
486 {
487 errno_t error = 0;
488 struct pktap_softc *__single pktap;
489
490 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
491
492 pktap = ifp->if_softc;
493 if (pktap == NULL) {
494 error = ENOENT;
495 printf("%s: pktap NULL - error %d\n", __func__, error);
496 goto done;
497 }
498
499 switch (ifd->ifd_cmd) {
500 case PKTP_CMD_FILTER_SET: {
501 struct pktap_filter user_filters[PKTAP_MAX_FILTERS];
502 int i;
503 int got_op_none = 0;
504
505 if (ifd->ifd_len != PKTAP_MAX_FILTERS * sizeof(struct pktap_filter)) {
506 printf("%s: PKTP_CMD_FILTER_SET bad ifd_len %llu - error %d\n",
507 __func__, ifd->ifd_len, error);
508 error = EINVAL;
509 break;
510 }
511 error = copyin(CAST_USER_ADDR_T(ifd->ifd_data), &user_filters, (size_t)ifd->ifd_len);
512 if (error) {
513 printf("%s: copyin - error %d\n", __func__, error);
514 goto done;
515 }
516 /*
517 * Validate user provided parameters
518 */
519 for (i = 0; i < PKTAP_MAX_FILTERS; i++) {
520 struct pktap_filter *x_filter = user_filters + i;
521
522 switch (x_filter->filter_op) {
523 case PKTAP_FILTER_OP_NONE:
524 /* Following entries must be PKTAP_FILTER_OP_NONE */
525 got_op_none = 1;
526 break;
527 case PKTAP_FILTER_OP_PASS:
528 case PKTAP_FILTER_OP_SKIP:
529 /* Invalid after PKTAP_FILTER_OP_NONE */
530 if (got_op_none) {
531 error = EINVAL;
532 break;
533 }
534 break;
535 default:
536 error = EINVAL;
537 break;
538 }
539 if (error != 0) {
540 break;
541 }
542
543 switch (x_filter->filter_param) {
544 case PKTAP_FILTER_OP_NONE:
545 if (x_filter->filter_op != PKTAP_FILTER_OP_NONE) {
546 error = EINVAL;
547 break;
548 }
549 break;
550
551 /*
552 * Do not allow to tap a pktap from a pktap
553 */
554 case PKTAP_FILTER_PARAM_IF_TYPE:
555 if (x_filter->filter_param_if_type == IFT_PKTAP ||
556 x_filter->filter_param_if_type > 0xff) {
557 error = EINVAL;
558 break;
559 }
560 break;
561
562 case PKTAP_FILTER_PARAM_IF_NAME:
563 if (strlcmp(x_filter->filter_param_if_name, PKTAP_IFNAME,
564 strlen(PKTAP_IFNAME)) == 0) {
565 error = EINVAL;
566 break;
567 }
568 break;
569
570 default:
571 error = EINVAL;
572 break;
573 }
574 if (error != 0) {
575 break;
576 }
577 }
578 if (error != 0) {
579 break;
580 }
581 for (i = 0; i < PKTAP_MAX_FILTERS; i++) {
582 struct kern_pktap_filter *pktap_filter = pktap->pktp_filters + i;
583 struct pktap_filter *x_filter = user_filters + i;
584
585 pktap_filter->filter_op = x_filter->filter_op;
586 pktap_filter->filter_param = x_filter->filter_param;
587
588 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_TYPE) {
589 pktap_filter->filter_param_if_type = x_filter->filter_param_if_type;
590 } else if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_NAME) {
591 size_t len;
592
593 strbufcpy(pktap_filter->filter_param_if_name,
594 x_filter->filter_param_if_name);
595 /*
596 * If name does not end with a number then it's a "wildcard" match
597 * where we compare the prefix of the interface name
598 */
599 len = strbuflen(pktap_filter->filter_param_if_name);
600 if (pktap_filter->filter_param_if_name[len - 1] < '0' ||
601 pktap_filter->filter_param_if_name[len - 1] > '9') {
602 pktap_filter->filter_ifname_prefix_match = true;
603 } else {
604 pktap_filter->filter_ifname_prefix_match = false;
605 }
606 pktap_filter->filter_ifname_len = len;
607 }
608 }
609 break;
610 }
611 default:
612 error = EINVAL;
613 break;
614 }
615
616 done:
617 return error;
618 }
619
620 __private_extern__ errno_t
pktap_ioctl(ifnet_t ifp,unsigned long cmd,void * __sized_by (IOCPARM_LEN (cmd))data)621 pktap_ioctl(ifnet_t ifp, unsigned long cmd, void *__sized_by(IOCPARM_LEN(cmd)) data)
622 {
623 errno_t error = 0;
624
625 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
626
627 if ((cmd & IOC_IN)) {
628 error = kauth_authorize_generic(kauth_cred_get(), KAUTH_GENERIC_ISSUSER);
629 if (error) {
630 PKTAP_LOG(PKTP_LOG_ERROR,
631 "%s: kauth_authorize_generic(KAUTH_GENERIC_ISSUSER) - error %d\n",
632 __func__, error);
633 goto done;
634 }
635 }
636
637 switch (cmd) {
638 case SIOCGDRVSPEC32: {
639 struct ifdrv64 ifd;
640 struct ifdrv32 *ifd32 = (struct ifdrv32 *)data;
641
642 memcpy(ifd.ifd_name, ifd32->ifd_name, sizeof(ifd.ifd_name));
643 ifd.ifd_cmd = ifd32->ifd_cmd;
644 ifd.ifd_len = ifd32->ifd_len;
645 ifd.ifd_data = ifd32->ifd_data;
646
647 error = pktap_getdrvspec(ifp, &ifd);
648
649 break;
650 }
651 case SIOCGDRVSPEC64: {
652 struct ifdrv64 *ifd64 = (struct ifdrv64 *)data;
653
654 error = pktap_getdrvspec(ifp, ifd64);
655
656 break;
657 }
658 case SIOCSDRVSPEC32: {
659 struct ifdrv64 ifd;
660 struct ifdrv32 *ifd32 = (struct ifdrv32 *)data;
661
662 memcpy(ifd.ifd_name, ifd32->ifd_name, sizeof(ifd.ifd_name));
663 ifd.ifd_cmd = ifd32->ifd_cmd;
664 ifd.ifd_len = ifd32->ifd_len;
665 ifd.ifd_data = ifd32->ifd_data;
666
667 error = pktap_setdrvspec(ifp, &ifd);
668 break;
669 }
670 case SIOCSDRVSPEC64: {
671 struct ifdrv64 *ifd64 = (struct ifdrv64 *)data;
672
673 error = pktap_setdrvspec(ifp, ifd64);
674
675 break;
676 }
677 default:
678 error = ENOTSUP;
679 break;
680 }
681 done:
682 return error;
683 }
684
685 __private_extern__ void
pktap_detach(ifnet_t ifp)686 pktap_detach(ifnet_t ifp)
687 {
688 struct pktap_softc *__single pktap;
689
690 PKTAP_LOG(PKTP_LOG_FUNC, "%s\n", ifp->if_xname);
691
692 lck_rw_lock_exclusive(&pktap_lck_rw);
693
694 pktap = ifp->if_softc;
695 ifp->if_softc = NULL;
696 LIST_REMOVE(pktap, pktp_link);
697
698 lck_rw_done(&pktap_lck_rw);
699
700 /* Drop reference as it's no more on the global list */
701 ifnet_release(ifp);
702
703 kfree_type(struct pktap_softc, pktap);
704 /* This is for the reference taken by ifnet_attach() */
705 (void) ifnet_release(ifp);
706 }
707
708 __private_extern__ int
pktap_filter_evaluate(struct pktap_softc * pktap,struct ifnet * ifp)709 pktap_filter_evaluate(struct pktap_softc *pktap, struct ifnet *ifp)
710 {
711 int i;
712 int result = PKTAP_FILTER_SKIP; /* Need positive matching rule to pass */
713 int match = 0;
714
715 for (i = 0; i < PKTAP_MAX_FILTERS; i++) {
716 struct kern_pktap_filter *pktap_filter = pktap->pktp_filters + i;
717
718 switch (pktap_filter->filter_op) {
719 case PKTAP_FILTER_OP_NONE:
720 match = 1;
721 break;
722
723 case PKTAP_FILTER_OP_PASS:
724 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_TYPE) {
725 if (pktap_filter->filter_param_if_type == 0 ||
726 ifp->if_type == pktap_filter->filter_param_if_type) {
727 result = PKTAP_FILTER_OK;
728 match = 1;
729 PKTAP_LOG(PKTP_LOG_FILTER, "pass %s match type %u\n",
730 ifp->if_xname, pktap_filter->filter_param_if_type);
731 break;
732 }
733 }
734 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_NAME) {
735 if (pktap_filter->filter_ifname_prefix_match == false) {
736 match = !strlcmp(pktap_filter->filter_param_if_name,
737 ifp->if_xname,
738 pktap_filter->filter_ifname_len);
739 } else {
740 match = strprefix(ifp->if_xname,
741 __unsafe_forge_null_terminated(char *, pktap_filter->filter_param_if_name));
742 }
743
744 if (match) {
745 result = PKTAP_FILTER_OK;
746 PKTAP_LOG(PKTP_LOG_FILTER, "pass %s match name %s\n",
747 ifp->if_xname, pktap_filter->filter_param_if_name);
748 break;
749 }
750 }
751 break;
752
753 case PKTAP_FILTER_OP_SKIP:
754 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_TYPE) {
755 if (pktap_filter->filter_param_if_type == 0 ||
756 ifp->if_type == pktap_filter->filter_param_if_type) {
757 result = PKTAP_FILTER_SKIP;
758 match = 1;
759 PKTAP_LOG(PKTP_LOG_FILTER, "skip %s match type %u\n",
760 ifp->if_xname, pktap_filter->filter_param_if_type);
761 break;
762 }
763 }
764 if (pktap_filter->filter_param == PKTAP_FILTER_PARAM_IF_NAME) {
765 if (pktap_filter->filter_ifname_prefix_match == false) {
766 match = !strlcmp(pktap_filter->filter_param_if_name,
767 ifp->if_xname,
768 pktap_filter->filter_ifname_len);
769 } else {
770 match = strprefix(ifp->if_xname,
771 __unsafe_forge_null_terminated(char *, pktap_filter->filter_param_if_name));
772 }
773 if (match) {
774 result = PKTAP_FILTER_SKIP;
775 PKTAP_LOG(PKTP_LOG_FILTER, "skip %s match name %s\n",
776 ifp->if_xname, pktap_filter->filter_param_if_name);
777 break;
778 }
779 }
780 break;
781 }
782 if (match) {
783 break;
784 }
785 }
786
787 if (match == 0) {
788 PKTAP_LOG(PKTP_LOG_FILTER, "%s no match\n",
789 ifp->if_xname);
790 }
791 return result;
792 }
793
794 static void
pktap_set_procinfo(struct pktap_header * hdr,struct so_procinfo * soprocinfo)795 pktap_set_procinfo(struct pktap_header *hdr, struct so_procinfo *soprocinfo)
796 {
797 hdr->pth_pid = soprocinfo->spi_pid;
798 if (hdr->pth_comm[0] == 0) {
799 proc_name(soprocinfo->spi_pid, hdr->pth_comm, MAXCOMLEN);
800 }
801 strbufcpy(hdr->pth_comm, soprocinfo->spi_proc_name);
802
803 if (soprocinfo->spi_pid != 0) {
804 uuid_copy(hdr->pth_uuid, soprocinfo->spi_uuid);
805 }
806
807 if (soprocinfo->spi_delegated != 0) {
808 hdr->pth_flags |= PTH_FLAG_PROC_DELEGATED;
809 hdr->pth_epid = soprocinfo->spi_epid;
810 strbufcpy(hdr->pth_ecomm, soprocinfo->spi_e_proc_name);
811 uuid_copy(hdr->pth_euuid, soprocinfo->spi_euuid);
812 }
813 }
814
815 __private_extern__ void
pktap_finalize_proc_info(struct pktap_header * hdr)816 pktap_finalize_proc_info(struct pktap_header *hdr)
817 {
818 int found;
819 struct so_procinfo soprocinfo;
820
821 if (!(hdr->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
822 return;
823 }
824
825 if (hdr->pth_ipproto == IPPROTO_TCP) {
826 found = inp_findinpcb_procinfo(&tcbinfo, hdr->pth_flowid,
827 &soprocinfo);
828 } else if (hdr->pth_ipproto == IPPROTO_UDP) {
829 found = inp_findinpcb_procinfo(&udbinfo, hdr->pth_flowid,
830 &soprocinfo);
831 } else {
832 found = inp_findinpcb_procinfo(&ripcbinfo, hdr->pth_flowid,
833 &soprocinfo);
834 }
835
836 if (found == 1) {
837 pktap_set_procinfo(hdr, &soprocinfo);
838 }
839 }
840
841 static void
pktap_v2_set_procinfo(struct pktap_v2_hdr * pktap_v2_hdr,struct so_procinfo * soprocinfo)842 pktap_v2_set_procinfo(struct pktap_v2_hdr *pktap_v2_hdr,
843 struct so_procinfo *soprocinfo)
844 {
845 uint8_t *region = __unsafe_forge_bidi_indexable(uint8_t *,
846 pktap_v2_hdr,
847 pktap_v2_hdr->pth_length);
848 pktap_v2_hdr->pth_pid = soprocinfo->spi_pid;
849
850 if (soprocinfo->spi_pid != 0 && soprocinfo->spi_pid != -1) {
851 if (pktap_v2_hdr->pth_comm_offset != 0) {
852 char *ptr = __unsafe_forge_bidi_indexable(char *,
853 region + pktap_v2_hdr->pth_comm_offset,
854 PKTAP_MAX_COMM_SIZE);
855
856 strbufcpy(ptr, PKTAP_MAX_COMM_SIZE, soprocinfo->spi_proc_name, PKTAP_MAX_COMM_SIZE);
857 }
858 if (pktap_v2_hdr->pth_uuid_offset != 0) {
859 uuid_t *ptr = __unsafe_forge_bidi_indexable(uuid_t *,
860 region + pktap_v2_hdr->pth_uuid_offset,
861 sizeof(uuid_t));
862 uuid_copy(*ptr, soprocinfo->spi_uuid);
863 }
864 }
865
866 if (!(pktap_v2_hdr->pth_flags & PTH_FLAG_PROC_DELEGATED)) {
867 return;
868 }
869
870 /*
871 * The effective UUID may be set independently from the effective pid
872 */
873 if (soprocinfo->spi_delegated != 0) {
874 pktap_v2_hdr->pth_flags |= PTH_FLAG_PROC_DELEGATED;
875 pktap_v2_hdr->pth_e_pid = soprocinfo->spi_epid;
876
877 if (soprocinfo->spi_pid != 0 && soprocinfo->spi_pid != -1 &&
878 pktap_v2_hdr->pth_e_comm_offset != 0) {
879 char *ptr = __unsafe_forge_bidi_indexable(char *,
880 region + pktap_v2_hdr->pth_e_comm_offset,
881 PKTAP_MAX_COMM_SIZE);
882 strbufcpy(ptr, PKTAP_MAX_COMM_SIZE, soprocinfo->spi_e_proc_name, PKTAP_MAX_COMM_SIZE);
883 }
884 if (pktap_v2_hdr->pth_e_uuid_offset != 0) {
885 uuid_t *ptr = __unsafe_forge_bidi_indexable(uuid_t *,
886 region + pktap_v2_hdr->pth_e_uuid_offset,
887 sizeof(uuid_t));
888 uuid_copy(*ptr, soprocinfo->spi_euuid);
889 }
890 }
891 }
892
893 __private_extern__ void
pktap_v2_finalize_proc_info(struct pktap_v2_hdr * pktap_v2_hdr)894 pktap_v2_finalize_proc_info(struct pktap_v2_hdr *pktap_v2_hdr)
895 {
896 int found;
897 struct so_procinfo soprocinfo;
898
899 if (!(pktap_v2_hdr->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
900 return;
901 }
902
903 if (pktap_v2_hdr->pth_ipproto == IPPROTO_TCP) {
904 found = inp_findinpcb_procinfo(&tcbinfo,
905 pktap_v2_hdr->pth_flowid, &soprocinfo);
906 } else if (pktap_v2_hdr->pth_ipproto == IPPROTO_UDP) {
907 found = inp_findinpcb_procinfo(&udbinfo,
908 pktap_v2_hdr->pth_flowid, &soprocinfo);
909 } else {
910 found = inp_findinpcb_procinfo(&ripcbinfo,
911 pktap_v2_hdr->pth_flowid, &soprocinfo);
912 }
913 if (found == 1) {
914 pktap_v2_set_procinfo(pktap_v2_hdr, &soprocinfo);
915 }
916 }
917
918 __private_extern__ void
pktap_fill_proc_info(struct pktap_header * hdr,protocol_family_t proto,struct mbuf * m,u_int32_t pre,int outgoing,struct ifnet * ifp)919 pktap_fill_proc_info(struct pktap_header *hdr, protocol_family_t proto,
920 struct mbuf *m, u_int32_t pre, int outgoing, struct ifnet *ifp)
921 {
922 /*
923 * Getting the pid and procname is expensive
924 * For outgoing, do the lookup only if there's an
925 * associated socket as indicated by the flowhash
926 */
927 if (outgoing != 0 && m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
928 /*
929 * To avoid lock ordering issues we delay the proc UUID lookup
930 * to the BPF read as we cannot
931 * assume the socket lock is unlocked on output
932 */
933 hdr->pth_flags |= PTH_FLAG_DELAY_PKTAP;
934 hdr->pth_flags |= PTH_FLAG_SOCKET;
935 hdr->pth_flowid = m->m_pkthdr.pkt_flowid;
936
937 if (m->m_pkthdr.pkt_flags & PKTF_FLOW_RAWSOCK) {
938 hdr->pth_ipproto = IPPROTO_RAW;
939 } else {
940 hdr->pth_ipproto = m->m_pkthdr.pkt_proto;
941 }
942
943 if (hdr->pth_ipproto == IPPROTO_TCP) {
944 hdr->pth_pid = m->m_pkthdr.tx_tcp_pid;
945 hdr->pth_epid = m->m_pkthdr.tx_tcp_e_pid;
946 } else if (hdr->pth_ipproto == IPPROTO_UDP) {
947 hdr->pth_pid = m->m_pkthdr.tx_udp_pid;
948 hdr->pth_epid = m->m_pkthdr.tx_udp_e_pid;
949 } else if (hdr->pth_ipproto == IPPROTO_RAW) {
950 hdr->pth_pid = m->m_pkthdr.tx_rawip_pid;
951 hdr->pth_epid = m->m_pkthdr.tx_rawip_e_pid;
952 }
953
954 if (hdr->pth_pid != 0 && hdr->pth_pid != -1) {
955 proc_name(hdr->pth_pid, hdr->pth_comm, MAXCOMLEN);
956 } else {
957 hdr->pth_pid = -1;
958 }
959
960 if (hdr->pth_epid != 0 && hdr->pth_epid != -1) {
961 hdr->pth_flags |= PTH_FLAG_PROC_DELEGATED;
962 proc_name(hdr->pth_epid, hdr->pth_ecomm, MAXCOMLEN);
963 } else {
964 hdr->pth_epid = -1;
965 }
966
967 if (m->m_pkthdr.pkt_flags & PKTF_NEW_FLOW) {
968 hdr->pth_flags |= PTH_FLAG_NEW_FLOW;
969 }
970 } else if (outgoing == 0) {
971 int found = 0;
972 struct so_procinfo soprocinfo;
973 struct inpcb *inp = NULL;
974
975 memset(&soprocinfo, 0, sizeof(struct so_procinfo));
976
977 if (proto == PF_INET) {
978 struct ip ip;
979 errno_t error;
980 size_t hlen;
981 struct in_addr faddr, laddr;
982 u_short fport = 0, lport = 0;
983 struct inpcbinfo *pcbinfo = NULL;
984 int wildcard = 0;
985
986 error = mbuf_copydata(m, pre, sizeof(struct ip), &ip);
987 if (error != 0) {
988 PKTAP_LOG(PKTP_LOG_ERROR,
989 "mbuf_copydata tcp v4 failed for %s\n",
990 hdr->pth_ifname);
991 goto done;
992 }
993 hlen = IP_VHL_HL(ip.ip_vhl) << 2;
994
995 faddr = ip.ip_src;
996 laddr = ip.ip_dst;
997
998 if (ip.ip_p == IPPROTO_TCP) {
999 struct tcphdr th;
1000
1001 error = mbuf_copydata(m, pre + hlen,
1002 sizeof(struct tcphdr), &th);
1003 if (error != 0) {
1004 goto done;
1005 }
1006
1007 fport = th.th_sport;
1008 lport = th.th_dport;
1009
1010 pcbinfo = &tcbinfo;
1011 } else if (ip.ip_p == IPPROTO_UDP) {
1012 struct udphdr uh;
1013
1014 error = mbuf_copydata(m, pre + hlen,
1015 sizeof(struct udphdr), &uh);
1016 if (error != 0) {
1017 PKTAP_LOG(PKTP_LOG_ERROR,
1018 "mbuf_copydata udp v4 failed for %s\n",
1019 hdr->pth_ifname);
1020 goto done;
1021 }
1022 fport = uh.uh_sport;
1023 lport = uh.uh_dport;
1024
1025 pcbinfo = &udbinfo;
1026 wildcard = 1;
1027 }
1028 if (pcbinfo != NULL) {
1029 inp = in_pcblookup_hash_try(pcbinfo, faddr,
1030 fport, laddr, lport, wildcard,
1031 outgoing ? NULL : ifp);
1032
1033 if (inp == NULL && hdr->pth_iftype != IFT_LOOP) {
1034 PKTAP_LOG(PKTP_LOG_NOPCB,
1035 "in_pcblookup_hash_try no pcb %s\n",
1036 hdr->pth_ifname);
1037 }
1038 } else {
1039 PKTAP_LOG(PKTP_LOG_NOPCB,
1040 "unknown ip_p %u on %s\n",
1041 ip.ip_p, hdr->pth_ifname);
1042 pktap_hexdump(PKTP_LOG_NOPCB, &ip, sizeof(struct ip));
1043 }
1044 } else if (proto == PF_INET6) {
1045 struct ip6_hdr ip6;
1046 errno_t error;
1047 struct in6_addr *faddr;
1048 struct in6_addr *laddr;
1049 u_short fport = 0, lport = 0;
1050 struct inpcbinfo *pcbinfo = NULL;
1051 int wildcard = 0;
1052
1053 error = mbuf_copydata(m, pre, sizeof(struct ip6_hdr), &ip6);
1054 if (error != 0) {
1055 goto done;
1056 }
1057
1058 faddr = &ip6.ip6_src;
1059 laddr = &ip6.ip6_dst;
1060
1061 if (ip6.ip6_nxt == IPPROTO_TCP) {
1062 struct tcphdr th;
1063
1064 error = mbuf_copydata(m, pre + sizeof(struct ip6_hdr),
1065 sizeof(struct tcphdr), &th);
1066 if (error != 0) {
1067 PKTAP_LOG(PKTP_LOG_ERROR,
1068 "mbuf_copydata tcp v6 failed for %s\n",
1069 hdr->pth_ifname);
1070 goto done;
1071 }
1072
1073 fport = th.th_sport;
1074 lport = th.th_dport;
1075
1076 pcbinfo = &tcbinfo;
1077 } else if (ip6.ip6_nxt == IPPROTO_UDP) {
1078 struct udphdr uh;
1079
1080 error = mbuf_copydata(m, pre + sizeof(struct ip6_hdr),
1081 sizeof(struct udphdr), &uh);
1082 if (error != 0) {
1083 PKTAP_LOG(PKTP_LOG_ERROR,
1084 "mbuf_copydata udp v6 failed for %s\n",
1085 hdr->pth_ifname);
1086 goto done;
1087 }
1088
1089 fport = uh.uh_sport;
1090 lport = uh.uh_dport;
1091
1092 pcbinfo = &udbinfo;
1093 wildcard = 1;
1094 }
1095 if (pcbinfo != NULL) {
1096 inp = in6_pcblookup_hash_try(pcbinfo, faddr, fport, ip6_input_getdstifscope(m),
1097 laddr, lport, ip6_input_getsrcifscope(m), wildcard, outgoing ? NULL : ifp);
1098
1099 if (inp == NULL && hdr->pth_iftype != IFT_LOOP) {
1100 PKTAP_LOG(PKTP_LOG_NOPCB,
1101 "in6_pcblookup_hash_try no pcb %s\n",
1102 hdr->pth_ifname);
1103 }
1104 } else {
1105 PKTAP_LOG(PKTP_LOG_NOPCB,
1106 "unknown ip6.ip6_nxt %u on %s\n",
1107 ip6.ip6_nxt, hdr->pth_ifname);
1108 pktap_hexdump(PKTP_LOG_NOPCB, &ip6, sizeof(struct ip6_hdr));
1109 }
1110 }
1111 if (inp != NULL) {
1112 hdr->pth_flags |= PTH_FLAG_SOCKET;
1113 if (inp->inp_state != INPCB_STATE_DEAD && inp->inp_socket != NULL) {
1114 found = 1;
1115 inp_get_soprocinfo(inp, &soprocinfo);
1116 }
1117 in_pcb_checkstate(inp, WNT_RELEASE, 0);
1118 }
1119 done:
1120 /*
1121 * -1 means PID not found
1122 */
1123 hdr->pth_pid = -1;
1124 hdr->pth_epid = -1;
1125
1126 if (found != 0) {
1127 pktap_set_procinfo(hdr, &soprocinfo);
1128 }
1129 }
1130 }
1131
1132 __private_extern__ void
pktap_bpf_tap(struct ifnet * ifp,protocol_family_t proto,struct mbuf * m,u_int32_t pre,u_int32_t post,int outgoing)1133 pktap_bpf_tap(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m,
1134 u_int32_t pre, u_int32_t post, int outgoing)
1135 {
1136 struct pktap_softc *__single pktap;
1137 void (*bpf_tap_func)(ifnet_t, u_int32_t, mbuf_t, void *, size_t) =
1138 outgoing ? bpf_tap_out : bpf_tap_in;
1139
1140 /*
1141 * Skip the coprocessor interface
1142 */
1143 if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) {
1144 return;
1145 }
1146
1147 lck_rw_lock_shared(&pktap_lck_rw);
1148
1149 /*
1150 * No need to take the ifnet_lock as the struct ifnet field if_bpf is
1151 * protected by the BPF subsystem
1152 */
1153 LIST_FOREACH(pktap, &pktap_list, pktp_link) {
1154 int filter_result;
1155
1156 filter_result = pktap_filter_evaluate(pktap, ifp);
1157 if (filter_result == PKTAP_FILTER_SKIP) {
1158 continue;
1159 }
1160
1161 if (pktap->pktp_dlt_raw_count > 0) {
1162 /* We accept only IPv4 and IPv6 packets for the raw DLT */
1163 if ((proto == AF_INET || proto == AF_INET6) &&
1164 !(m->m_pkthdr.pkt_flags & PKTF_INET_RESOLVE)) {
1165 /*
1166 * We can play just with the length of the first mbuf in the
1167 * chain because bpf_tap_imp() disregard the packet length
1168 * of the mbuf packet header.
1169 */
1170 if (mbuf_setdata(m, m_mtod_current(m) + pre, m->m_len - pre) == 0) {
1171 bpf_tap_func(pktap->pktp_ifp, DLT_RAW, m, NULL, 0);
1172 mbuf_setdata(m, m_mtod_current(m) - pre, m->m_len + pre);
1173 }
1174 }
1175 }
1176
1177 if (pktap->pktp_dlt_pkttap_count > 0) {
1178 struct {
1179 struct pktap_header hdr;
1180 u_int32_t proto;
1181 } hdr_buffer;
1182 struct pktap_header *hdr = &hdr_buffer.hdr;
1183 size_t hdr_size = sizeof(struct pktap_header);
1184 int unknown_if_type = 0;
1185 size_t data_adjust = 0;
1186 u_int32_t pre_adjust = 0;
1187
1188 /* Verify the structure is packed */
1189 _CASSERT(sizeof(hdr_buffer) == sizeof(struct pktap_header) + sizeof(u_int32_t));
1190
1191 bzero(&hdr_buffer, sizeof(hdr_buffer));
1192 hdr->pth_length = sizeof(struct pktap_header);
1193 hdr->pth_type_next = PTH_TYPE_PACKET;
1194
1195 /*
1196 * Set DLT of packet based on interface type
1197 */
1198 switch (ifp->if_type) {
1199 case IFT_LOOP:
1200 case IFT_GIF:
1201 case IFT_STF:
1202 case IFT_CELLULAR:
1203 /*
1204 * Packets from pdp interfaces have no loopback
1205 * header that contain the protocol number.
1206 * As BPF just concatenate the header and the
1207 * packet content in a single buffer,
1208 * stash the protocol after the pktap header
1209 * and adjust the size of the header accordingly
1210 */
1211 hdr->pth_dlt = DLT_NULL;
1212 if (pre == 0) {
1213 hdr_buffer.proto = proto;
1214 hdr_size = sizeof(hdr_buffer);
1215 pre_adjust = sizeof(hdr_buffer.proto);
1216 }
1217 break;
1218 case IFT_ETHER:
1219 case IFT_BRIDGE:
1220 case IFT_L2VLAN:
1221 case IFT_IEEE8023ADLAG:
1222 hdr->pth_dlt = DLT_EN10MB;
1223 break;
1224 case IFT_PPP:
1225 hdr->pth_dlt = DLT_PPP;
1226 break;
1227 case IFT_IEEE1394:
1228 hdr->pth_dlt = DLT_APPLE_IP_OVER_IEEE1394;
1229 break;
1230 case IFT_OTHER:
1231 if (ifp->if_family == IFNET_FAMILY_IPSEC ||
1232 ifp->if_family == IFNET_FAMILY_UTUN) {
1233 /*
1234 * For utun:
1235 * - incoming packets do not have the prefix set to four
1236 * - some packets are as small as two bytes!
1237 */
1238 if (m_pktlen(m) < 4) {
1239 goto done;
1240 }
1241 if (proto != AF_INET && proto != AF_INET6) {
1242 goto done;
1243 }
1244 if (proto == AF_INET && (size_t) m_pktlen(m) - 4 < sizeof(struct ip)) {
1245 goto done;
1246 }
1247 if (proto == AF_INET6 && (size_t) m_pktlen(m) - 4 < sizeof(struct ip6_hdr)) {
1248 goto done;
1249 }
1250
1251 /*
1252 * Handle two cases:
1253 * - The old utun encapsulation with the protocol family in network order
1254 * - A raw IPv4 or IPv6 packet
1255 */
1256 uint8_t data = *mtod(m, uint8_t *);
1257 if ((data >> 4) == 4 || (data >> 4) == 6) {
1258 pre = 4;
1259 } else {
1260 /*
1261 * Skip the protocol in the mbuf as it's in network order
1262 */
1263 pre = 4;
1264 data_adjust = 4;
1265 }
1266 }
1267 hdr->pth_dlt = DLT_NULL;
1268 hdr_buffer.proto = proto;
1269 hdr_size = sizeof(hdr_buffer);
1270 break;
1271 default:
1272 if (pre == 0) {
1273 hdr->pth_dlt = DLT_RAW;
1274 } else {
1275 unknown_if_type = 1;
1276 }
1277 break;
1278 }
1279 if (unknown_if_type) {
1280 PKTAP_LOG(PKTP_LOG_FUNC,
1281 "unknown if_type %u for %s\n",
1282 ifp->if_type, ifp->if_xname);
1283 pktap_count_unknown_if_type += 1;
1284 } else {
1285 strlcpy(hdr->pth_ifname, ifp->if_xname,
1286 sizeof(hdr->pth_ifname));
1287 hdr->pth_flags |= outgoing ? PTH_FLAG_DIR_OUT : PTH_FLAG_DIR_IN;
1288 hdr->pth_protocol_family = proto;
1289 hdr->pth_frame_pre_length = pre + pre_adjust;
1290 hdr->pth_frame_post_length = post;
1291 hdr->pth_iftype = ifp->if_type;
1292 hdr->pth_ifunit = ifp->if_unit;
1293
1294 if (m->m_pkthdr.pkt_flags & PKTF_KEEPALIVE) {
1295 hdr->pth_flags |= PTH_FLAG_KEEP_ALIVE;
1296 }
1297 if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) {
1298 hdr->pth_flags |= PTH_FLAG_REXMIT;
1299 }
1300 if (m->m_pkthdr.pkt_flags & PKTF_WAKE_PKT) {
1301 hdr->pth_flags |= PTH_FLAG_WAKE_PKT;
1302 }
1303 if (outgoing != 0) {
1304 hdr->pth_comp_gencnt = m->m_pkthdr.comp_gencnt;
1305 }
1306
1307 pktap_fill_proc_info(hdr, proto, m, pre, outgoing, ifp);
1308
1309 hdr->pth_svc = so_svc2tc(m->m_pkthdr.pkt_svc);
1310
1311 if (data_adjust == 0) {
1312 bpf_tap_func(pktap->pktp_ifp, DLT_PKTAP, m, &hdr_buffer,
1313 hdr_size);
1314 } else {
1315 /*
1316 * We can play just with the length of the first mbuf in the
1317 * chain because bpf_tap_imp() disregard the packet length
1318 * of the mbuf packet header.
1319 */
1320 if (mbuf_setdata(m, m_mtod_current(m) + data_adjust, m->m_len - data_adjust) == 0) {
1321 bpf_tap_func(pktap->pktp_ifp, DLT_PKTAP, m, &hdr_buffer,
1322 hdr_size);
1323 mbuf_setdata(m, m_mtod_current(m) - data_adjust, m->m_len + data_adjust);
1324 }
1325 }
1326 }
1327 }
1328 }
1329 done:
1330 lck_rw_done(&pktap_lck_rw);
1331 }
1332
1333 __private_extern__ void
pktap_input(struct ifnet * ifp,protocol_family_t proto,struct mbuf * m,char * frame_header)1334 pktap_input(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m,
1335 char *frame_header)
1336 {
1337 char *hdr;
1338 char *start;
1339
1340 /* Fast path */
1341 if (pktap_total_tap_count == 0 ||
1342 (m->m_pkthdr.pkt_flags & PKTF_SKIP_PKTAP) != 0) {
1343 return;
1344 }
1345
1346 start = m_mtod_lower_bound(m);
1347 hdr = mtod(m, char *);
1348 /* Make sure the frame header is fully contained in the mbuf */
1349 if (frame_header != NULL && frame_header >= start && frame_header <= hdr) {
1350 size_t o_len = m->m_len;
1351 u_int32_t pre = (u_int32_t)(hdr - frame_header);
1352
1353 if (mbuf_setdata(m, frame_header, o_len + pre) == 0) {
1354 PKTAP_LOG(PKTP_LOG_INPUT, "ifp %s proto %u pre %u post %u\n",
1355 ifp->if_xname, proto, pre, 0);
1356
1357 pktap_bpf_tap(ifp, proto, m, pre, 0, 0);
1358 mbuf_setdata(m, hdr, o_len);
1359 }
1360 } else {
1361 PKTAP_LOG(PKTP_LOG_INPUT, "ifp %s proto %u pre %u post %u\n",
1362 ifp->if_xname, proto, 0, 0);
1363
1364 pktap_bpf_tap(ifp, proto, m, 0, 0, 0);
1365 }
1366 }
1367
1368 __private_extern__ void
pktap_output(struct ifnet * ifp,protocol_family_t proto,struct mbuf * m,u_int32_t pre,u_int32_t post)1369 pktap_output(struct ifnet *ifp, protocol_family_t proto, struct mbuf *m,
1370 u_int32_t pre, u_int32_t post)
1371 {
1372 /* Fast path */
1373 if (pktap_total_tap_count == 0 ||
1374 (m->m_pkthdr.pkt_flags & PKTF_SKIP_PKTAP) != 0) {
1375 return;
1376 }
1377
1378 PKTAP_LOG(PKTP_LOG_OUTPUT, "ifp %s proto %u pre %u post %u\n",
1379 ifp->if_xname, proto, pre, post);
1380
1381 pktap_bpf_tap(ifp, proto, m, pre, post, 1);
1382 }
1383
1384 #if SKYWALK
1385
1386 typedef void (*tap_packet_func)(ifnet_t interface, u_int32_t dlt,
1387 kern_packet_t packet, void *__sized_by(header_len) header, size_t header_len);
1388
1389 static void
pktap_bpf_tap_packet(struct ifnet * ifp,protocol_family_t proto,uint32_t dlt,pid_t pid,const char * pname,pid_t epid,const char * epname,kern_packet_t pkt,const void * __sized_by (header_length)header,size_t header_length,uint8_t ipproto,uint32_t flowid,uint32_t flags,tap_packet_func tap_func)1390 pktap_bpf_tap_packet(struct ifnet *ifp, protocol_family_t proto, uint32_t dlt,
1391 pid_t pid, const char * pname, pid_t epid, const char * epname,
1392 kern_packet_t pkt, const void *__sized_by(header_length) header, size_t header_length,
1393 uint8_t ipproto, uint32_t flowid, uint32_t flags, tap_packet_func tap_func)
1394 {
1395 struct {
1396 struct pktap_header pkth;
1397 union {
1398 uint8_t llhdr[16];
1399 uint32_t proto;
1400 } extra;
1401 } hdr_buffer;
1402 struct pktap_header *hdr;
1403 size_t hdr_size;
1404 struct pktap_softc *__single pktap;
1405 uint32_t pre_length = 0;
1406
1407 /*
1408 * Skip the coprocessor interface
1409 */
1410 if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) {
1411 return;
1412 }
1413
1414 if (proto != AF_INET && proto != AF_INET6) {
1415 PKTAP_LOG(PKTP_LOG_ERROR,
1416 "unsupported protocol %d\n",
1417 proto);
1418 return;
1419 }
1420
1421 /* assume that we'll be tapping using PKTAP */
1422 hdr = &hdr_buffer.pkth;
1423 bzero(&hdr_buffer, sizeof(hdr_buffer));
1424 hdr->pth_length = sizeof(struct pktap_header);
1425 hdr->pth_type_next = PTH_TYPE_PACKET;
1426 hdr->pth_dlt = dlt;
1427 hdr->pth_pid = pid;
1428 if (pid != epid) {
1429 hdr->pth_epid = epid;
1430 } else {
1431 hdr->pth_epid = -1;
1432 }
1433 if (pname != NULL) {
1434 strlcpy(hdr->pth_comm, pname, sizeof(hdr->pth_comm));
1435 }
1436 if (epname != NULL) {
1437 strlcpy(hdr->pth_ecomm, epname, sizeof(hdr->pth_ecomm));
1438 }
1439 strlcpy(hdr->pth_ifname, ifp->if_xname, sizeof(hdr->pth_ifname));
1440 hdr->pth_flags |= flags;
1441 hdr->pth_ipproto = ipproto;
1442 hdr->pth_flowid = flowid;
1443 /*
1444 * Do the same as pktap_fill_proc_info() to defer looking up inpcb.
1445 * We do it for both inbound and outbound packets unlike the mbuf case.
1446 */
1447 if ((flags & PTH_FLAG_SOCKET) != 0 && ipproto != 0 && flowid != 0) {
1448 hdr->pth_flags |= PTH_FLAG_DELAY_PKTAP;
1449 }
1450 if (kern_packet_get_wake_flag(pkt)) {
1451 hdr->pth_flags |= PTH_FLAG_WAKE_PKT;
1452 }
1453 kern_packet_get_compression_generation_count(pkt, &hdr->pth_comp_gencnt);
1454 hdr->pth_trace_tag = kern_packet_get_trace_tag(pkt);
1455 hdr->pth_protocol_family = proto;
1456 hdr->pth_svc = so_svc2tc((mbuf_svc_class_t)
1457 kern_packet_get_service_class(pkt));
1458 hdr->pth_iftype = ifp->if_type;
1459 hdr->pth_ifunit = ifp->if_unit;
1460 hdr_size = sizeof(struct pktap_header);
1461 if (header != NULL && header_length != 0) {
1462 if (header_length > sizeof(hdr_buffer.extra.llhdr)) {
1463 PKTAP_LOG(PKTP_LOG_ERROR,
1464 "%s: header %d > %d\n",
1465 if_name(ifp), (int)header_length,
1466 (int)sizeof(hdr_buffer.extra.llhdr));
1467 return;
1468 }
1469 bcopy(header, hdr_buffer.extra.llhdr, header_length);
1470 hdr_size += header_length;
1471 pre_length = (uint32_t)header_length;
1472 } else if (dlt == DLT_RAW) {
1473 /*
1474 * Use the same DLT as has been used for the mbuf path
1475 */
1476 hdr->pth_dlt = DLT_NULL;
1477 hdr_buffer.extra.proto = proto;
1478 hdr_size = sizeof(struct pktap_header) + sizeof(u_int32_t);
1479 pre_length = sizeof(hdr_buffer.extra.proto);
1480 } else if (dlt == DLT_EN10MB) {
1481 pre_length = ETHER_HDR_LEN;
1482 }
1483 hdr->pth_frame_pre_length = pre_length;
1484
1485 lck_rw_lock_shared(&pktap_lck_rw);
1486 /*
1487 * No need to take the ifnet_lock as the struct ifnet field if_bpf is
1488 * protected by the BPF subsystem
1489 */
1490 LIST_FOREACH(pktap, &pktap_list, pktp_link) {
1491 int filter_result;
1492
1493 filter_result = pktap_filter_evaluate(pktap, ifp);
1494 if (filter_result == PKTAP_FILTER_SKIP) {
1495 continue;
1496 }
1497
1498 if (dlt == DLT_RAW && pktap->pktp_dlt_raw_count > 0) {
1499 (*tap_func)(pktap->pktp_ifp, DLT_RAW, pkt, NULL, 0);
1500 }
1501 if (pktap->pktp_dlt_pkttap_count > 0) {
1502 (*tap_func)(pktap->pktp_ifp, DLT_PKTAP,
1503 pkt, &hdr_buffer, hdr_size);
1504 }
1505 }
1506 lck_rw_done(&pktap_lck_rw);
1507 }
1508
1509 void
pktap_input_packet(struct ifnet * ifp,protocol_family_t proto,uint32_t dlt,pid_t pid,const char * pname,pid_t epid,const char * epname,kern_packet_t pkt,const void * __sized_by (header_length)header,size_t header_length,uint8_t ipproto,uint32_t flowid,uint32_t flags)1510 pktap_input_packet(struct ifnet *ifp, protocol_family_t proto, uint32_t dlt,
1511 pid_t pid, const char * pname, pid_t epid, const char * epname,
1512 kern_packet_t pkt, const void *__sized_by(header_length) header, size_t header_length,
1513 uint8_t ipproto, uint32_t flowid, uint32_t flags)
1514 {
1515 /* Fast path */
1516 if (pktap_total_tap_count == 0) {
1517 return;
1518 }
1519
1520 PKTAP_LOG(PKTP_LOG_INPUT, "IN %s proto %u pid %d epid %d\n",
1521 ifp->if_xname, proto, pid, epid);
1522 pktap_bpf_tap_packet(ifp, proto, dlt, pid, pname, epid, epname, pkt,
1523 header, header_length, ipproto, flowid,
1524 PTH_FLAG_DIR_IN | (flags & ~(PTH_FLAG_DIR_IN | PTH_FLAG_DIR_OUT)),
1525 bpf_tap_packet_in);
1526 }
1527
1528 void
pktap_output_packet(struct ifnet * ifp,protocol_family_t proto,uint32_t dlt,pid_t pid,const char * pname,pid_t epid,const char * epname,kern_packet_t pkt,const void * __sized_by (header_length)header,size_t header_length,uint8_t ipproto,uint32_t flowid,uint32_t flags)1529 pktap_output_packet(struct ifnet *ifp, protocol_family_t proto, uint32_t dlt,
1530 pid_t pid, const char * pname, pid_t epid, const char * epname,
1531 kern_packet_t pkt, const void *__sized_by(header_length) header, size_t header_length,
1532 uint8_t ipproto, uint32_t flowid, uint32_t flags)
1533 {
1534 /* Fast path */
1535 if (pktap_total_tap_count == 0) {
1536 return;
1537 }
1538
1539 PKTAP_LOG(PKTP_LOG_OUTPUT, "OUT %s proto %u pid %d epid %d\n",
1540 ifp->if_xname, proto, pid, epid);
1541 pktap_bpf_tap_packet(ifp, proto, dlt, pid, pname, epid, epname, pkt,
1542 header, header_length, ipproto, flowid,
1543 PTH_FLAG_DIR_OUT | (flags & ~(PTH_FLAG_DIR_IN | PTH_FLAG_DIR_OUT)),
1544 bpf_tap_packet_out);
1545 }
1546
1547 #endif /* SKYWALK */
1548
1549 void
convert_to_pktap_header_to_v2(struct bpf_packet * bpf_pkt,bool truncate)1550 convert_to_pktap_header_to_v2(struct bpf_packet *bpf_pkt, bool truncate)
1551 {
1552 struct pktap_header *pktap_header;
1553 size_t extra_src_size;
1554 struct pktap_buffer_v2_hdr_extra pktap_buffer_v2_hdr_extra;
1555 struct pktap_v2_hdr_space *pktap_v2_hdr_space;
1556 struct pktap_v2_hdr *pktap_v2_hdr;
1557 uint8_t *ptr;
1558
1559 pktap_header = (struct pktap_header *)bpf_pkt->bpfp_header;
1560
1561 if (pktap_header->pth_type_next != PTH_TYPE_PACKET) {
1562 return;
1563 }
1564
1565 VERIFY(bpf_pkt->bpfp_header_length >= sizeof(struct pktap_header));
1566
1567 /*
1568 * extra_src_size is the length of the optional link layer header
1569 */
1570 extra_src_size = bpf_pkt->bpfp_header_length -
1571 sizeof(struct pktap_header);
1572
1573 VERIFY(extra_src_size <= sizeof(union pktap_header_extra));
1574
1575 pktap_v2_hdr_space = &pktap_buffer_v2_hdr_extra.hdr_space;
1576 pktap_v2_hdr = &pktap_v2_hdr_space->pth_hdr;
1577 ptr = (uint8_t*) &pktap_buffer_v2_hdr_extra + sizeof(*pktap_v2_hdr);
1578
1579 COPY_PKTAP_COMMON_FIELDS_TO_V2(pktap_v2_hdr, pktap_header);
1580
1581 /*
1582 * When truncating don't bother with the process UUIDs
1583 */
1584 if (!truncate) {
1585 if ((pktap_header->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
1586 pktap_v2_hdr->pth_uuid_offset = pktap_v2_hdr->pth_length;
1587 pktap_v2_hdr->pth_length += sizeof(uuid_t);
1588 uuid_clear(*(uuid_t *)ptr);
1589 ptr += sizeof(uuid_t);
1590 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1591 } else if (!uuid_is_null(pktap_header->pth_uuid)) {
1592 pktap_v2_hdr->pth_uuid_offset = pktap_v2_hdr->pth_length;
1593 uuid_copy(*(uuid_t *)ptr, pktap_header->pth_uuid);
1594 pktap_v2_hdr->pth_length += sizeof(uuid_t);
1595 ptr += sizeof(uuid_t);
1596 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1597 }
1598
1599 if ((pktap_header->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
1600 if (pktap_header->pth_flags & PTH_FLAG_PROC_DELEGATED) {
1601 pktap_v2_hdr->pth_e_uuid_offset = pktap_v2_hdr->pth_length;
1602 uuid_clear(*(uuid_t *)ptr);
1603 pktap_v2_hdr->pth_length += sizeof(uuid_t);
1604 ptr += sizeof(uuid_t);
1605 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1606 }
1607 } else if (!uuid_is_null(pktap_header->pth_euuid)) {
1608 pktap_v2_hdr->pth_e_uuid_offset = pktap_v2_hdr->pth_length;
1609 uuid_copy(*(uuid_t *)ptr, pktap_header->pth_euuid);
1610 pktap_v2_hdr->pth_length += sizeof(uuid_t);
1611 ptr += sizeof(uuid_t);
1612 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1613 }
1614 }
1615
1616 if (pktap_header->pth_ifname[0] != 0) {
1617 size_t strsize;
1618
1619 pktap_v2_hdr->pth_ifname_offset = pktap_v2_hdr->pth_length;
1620
1621 /*
1622 * Note: strlcpy() returns the length of the string so we need
1623 * to add one for the end-of-string
1624 */
1625 size_t remaining_space = (uintptr_t)(pktap_v2_hdr_space + 1) - (uintptr_t)ptr;
1626 strsize = 1 + strlen(strbufcpy((char *)ptr,
1627 remaining_space,
1628 pktap_header->pth_ifname,
1629 sizeof(pktap_v2_hdr_space->pth_ifname)));
1630 pktap_v2_hdr->pth_length += strsize;
1631 ptr += strsize;
1632 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1633 }
1634
1635 /*
1636 * Do not waste space with the process name if we do not have a pid
1637 */
1638 if (pktap_header->pth_pid != 0 && pktap_header->pth_pid != -1) {
1639 if (pktap_header->pth_comm[0] != 0) {
1640 size_t strsize;
1641
1642 pktap_v2_hdr->pth_comm_offset = pktap_v2_hdr->pth_length;
1643
1644 size_t remaining_space = (uintptr_t)(pktap_v2_hdr_space + 1) - (uintptr_t)ptr;
1645 strsize = 1 + strlen(strbufcpy((char *)ptr, remaining_space,
1646 pktap_header->pth_comm,
1647 sizeof(pktap_v2_hdr_space->pth_comm)));
1648 pktap_v2_hdr->pth_length += strsize;
1649 ptr += strsize;
1650 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1651 } else if ((pktap_header->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
1652 size_t strsize = sizeof(pktap_v2_hdr_space->pth_comm);
1653
1654 pktap_v2_hdr->pth_comm_offset = pktap_v2_hdr->pth_length;
1655
1656 *ptr = 0; /* empty string by default */
1657 pktap_v2_hdr->pth_length += strsize;
1658 ptr += strsize;
1659 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1660 }
1661 }
1662
1663 /*
1664 * Do not waste space with the effective process name if we do not have
1665 * an effective pid or it's the same as the pid
1666 */
1667 if (pktap_header->pth_epid != 0 && pktap_header->pth_epid != -1 &&
1668 pktap_header->pth_epid != pktap_header->pth_pid) {
1669 if (pktap_header->pth_ecomm[0] != 0) {
1670 size_t strsize;
1671
1672 pktap_v2_hdr->pth_e_comm_offset = pktap_v2_hdr->pth_length;
1673
1674 size_t remaining_space = (uintptr_t)(pktap_v2_hdr_space + 1) - (uintptr_t)ptr;
1675
1676 strsize = 1 + strlen(strbufcpy((char *)ptr,
1677 remaining_space,
1678 pktap_header->pth_ecomm,
1679 sizeof(pktap_v2_hdr_space->pth_e_comm)));
1680 pktap_v2_hdr->pth_length += strsize;
1681 ptr += strsize;
1682 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1683 } else if ((pktap_header->pth_flags & PTH_FLAG_DELAY_PKTAP)) {
1684 size_t strsize = sizeof(pktap_v2_hdr_space->pth_e_comm);
1685
1686 pktap_v2_hdr->pth_e_comm_offset = pktap_v2_hdr->pth_length;
1687 *ptr = 0; /* empty string by default */
1688 pktap_v2_hdr->pth_length += strsize;
1689 ptr += strsize;
1690 VERIFY((void *)ptr < (void *)(pktap_v2_hdr_space + 1));
1691 }
1692 }
1693
1694 if (extra_src_size > 0) {
1695 uint8_t *extra_src_ptr = (uint8_t *)bpf_pkt->bpfp_header + sizeof(*pktap_header);
1696 uint8_t *extra_dst_ptr = (uint8_t *)&pktap_buffer_v2_hdr_extra + pktap_v2_hdr->pth_length;
1697
1698 VERIFY(pktap_v2_hdr->pth_length + extra_src_size <=
1699 sizeof(struct pktap_buffer_v2_hdr_extra));
1700
1701 memcpy(extra_dst_ptr, extra_src_ptr, extra_src_size);
1702 }
1703
1704 VERIFY(pktap_v2_hdr->pth_length + extra_src_size <=
1705 bpf_pkt->bpfp_header_length);
1706
1707 memcpy(bpf_pkt->bpfp_header, &pktap_buffer_v2_hdr_extra,
1708 pktap_v2_hdr->pth_length + extra_src_size);
1709 /*
1710 * For -fbounds-safety, we override the length which
1711 * is fragile. However, this is only called from
1712 * BPF which passes us a buffer allocated on the stack
1713 * and, in practice, that isn't going to cause any problems.
1714 */
1715 bpf_pkt->bpfp_header = bpf_pkt->bpfp_header;
1716 bpf_pkt->bpfp_header_length += pktap_v2_hdr->pth_length -
1717 sizeof(struct pktap_header);
1718
1719 bpf_pkt->bpfp_total_length += pktap_v2_hdr->pth_length -
1720 sizeof(struct pktap_header);
1721 }
1722