1 /*
2 * Copyright (c) 2000-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1990, 1991, 1993
30 * The Regents of the University of California. All rights reserved.
31 *
32 * This code is derived from the Stanford/CMU enet packet filter,
33 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
34 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
35 * Berkeley Laboratory.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * 3. All advertising materials mentioning features or use of this software
46 * must display the following acknowledgement:
47 * This product includes software developed by the University of
48 * California, Berkeley and its contributors.
49 * 4. Neither the name of the University nor the names of its contributors
50 * may be used to endorse or promote products derived from this software
51 * without specific prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 *
65 * @(#)bpf.c 8.2 (Berkeley) 3/28/94
66 *
67 * $FreeBSD: src/sys/net/bpf.c,v 1.59.2.5 2001/01/05 04:49:09 jdp Exp $
68 */
69 /*
70 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
71 * support for mandatory and extensible security protections. This notice
72 * is included in support of clause 2.2 (b) of the Apple Public License,
73 * Version 2.0.
74 */
75
76 #include "bpf.h"
77
78 #ifndef __GNUC__
79 #define inline
80 #else
81 #define inline __inline
82 #endif
83
84 #include <sys/param.h>
85 #include <sys/systm.h>
86 #include <sys/conf.h>
87 #include <sys/malloc.h>
88 #include <sys/mbuf.h>
89 #include <sys/time.h>
90 #include <sys/proc.h>
91 #include <sys/signalvar.h>
92 #include <sys/filio.h>
93 #include <sys/sockio.h>
94 #include <sys/ttycom.h>
95 #include <sys/filedesc.h>
96 #include <sys/uio_internal.h>
97 #include <sys/file_internal.h>
98 #include <sys/event.h>
99
100 #include <sys/poll.h>
101
102 #include <sys/socket.h>
103 #include <sys/socketvar.h>
104 #include <sys/vnode.h>
105
106 #include <net/if.h>
107 #include <net/bpf.h>
108 #include <net/bpfdesc.h>
109
110 #include <netinet/in.h>
111 #include <netinet/ip.h>
112 #include <netinet/ip6.h>
113 #include <netinet/in_pcb.h>
114 #include <netinet/in_var.h>
115 #include <netinet/ip_var.h>
116 #include <netinet/tcp.h>
117 #include <netinet/tcp_var.h>
118 #include <netinet/udp.h>
119 #include <netinet/udp_var.h>
120 #include <netinet/if_ether.h>
121 #include <netinet/isakmp.h>
122 #include <netinet6/esp.h>
123 #include <sys/kernel.h>
124 #include <sys/sysctl.h>
125 #include <net/firewire.h>
126
127 #include <miscfs/devfs/devfs.h>
128 #include <net/dlil.h>
129 #include <net/pktap.h>
130
131 #include <kern/assert.h>
132 #include <kern/locks.h>
133 #include <kern/thread_call.h>
134 #include <libkern/section_keywords.h>
135
136 #include <os/log.h>
137
138 #include <IOKit/IOBSD.h>
139
140 extern int tvtohz(struct timeval *);
141 extern char *proc_name_address(void *p);
142
143 #define BPF_BUFSIZE 4096
144 #define UIOMOVE(cp, len, code, uio) uiomove(cp, len, uio)
145
146 #define PRINET 26 /* interruptible */
147
148 #define ISAKMP_HDR_SIZE (sizeof(struct isakmp) + sizeof(struct isakmp_gen))
149 #define ESP_HDR_SIZE sizeof(struct newesp)
150
151 typedef void (*pktcopyfunc_t)(const void *, void *, size_t);
152
153 /*
154 * The default read buffer size is patchable.
155 */
156 static unsigned int bpf_bufsize = BPF_BUFSIZE;
157 SYSCTL_INT(_debug, OID_AUTO, bpf_bufsize, CTLFLAG_RW | CTLFLAG_LOCKED,
158 &bpf_bufsize, 0, "");
159
160 __private_extern__ unsigned int bpf_maxbufsize = BPF_MAXBUFSIZE;
161 static int sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS;
162 SYSCTL_PROC(_debug, OID_AUTO, bpf_maxbufsize, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
163 &bpf_maxbufsize, 0,
164 sysctl_bpf_maxbufsize, "I", "Default BPF max buffer size");
165
166 extern const int copysize_limit_panic;
167 #define BPF_BUFSIZE_CAP (copysize_limit_panic >> 1)
168 static int sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS;
169 SYSCTL_PROC(_debug, OID_AUTO, bpf_bufsize_cap, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
170 0, 0,
171 sysctl_bpf_bufsize_cap, "I", "Upper limit on BPF max buffer size");
172
173 static unsigned int bpf_maxdevices = 256;
174 SYSCTL_UINT(_debug, OID_AUTO, bpf_maxdevices, CTLFLAG_RD | CTLFLAG_LOCKED,
175 &bpf_maxdevices, 0, "");
176 /*
177 * bpf_wantpktap controls the defaul visibility of DLT_PKTAP
178 * For OS X is off by default so process need to use the ioctl BPF_WANT_PKTAP
179 * explicitly to be able to use DLT_PKTAP.
180 */
181 #if !XNU_TARGET_OS_OSX
182 static unsigned int bpf_wantpktap = 1;
183 #else /* XNU_TARGET_OS_OSX */
184 static unsigned int bpf_wantpktap = 0;
185 #endif /* XNU_TARGET_OS_OSX */
186 SYSCTL_UINT(_debug, OID_AUTO, bpf_wantpktap, CTLFLAG_RW | CTLFLAG_LOCKED,
187 &bpf_wantpktap, 0, "");
188
189 static int bpf_debug = 0;
190 SYSCTL_INT(_debug, OID_AUTO, bpf_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
191 &bpf_debug, 0, "");
192
193 static unsigned long bpf_trunc_overflow = 0;
194 SYSCTL_ULONG(_debug, OID_AUTO, bpf_trunc_overflow, CTLFLAG_RD | CTLFLAG_LOCKED,
195 &bpf_trunc_overflow, "");
196
197 static int bpf_hdr_comp_enable = 1;
198 SYSCTL_INT(_debug, OID_AUTO, bpf_hdr_comp_enable, CTLFLAG_RW | CTLFLAG_LOCKED,
199 &bpf_hdr_comp_enable, 1, "");
200
201 static int sysctl_bpf_stats SYSCTL_HANDLER_ARGS;
202 SYSCTL_PROC(_debug, OID_AUTO, bpf_stats, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
203 0, 0,
204 sysctl_bpf_stats, "S", "BPF statistics");
205
206 /*
207 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
208 * bpf_dtab holds pointer to the descriptors, indexed by minor device #
209 */
210 static struct bpf_if *bpf_iflist;
211 /*
212 * BSD now stores the bpf_d in the dev_t which is a struct
213 * on their system. Our dev_t is an int, so we still store
214 * the bpf_d in a separate table indexed by minor device #.
215 *
216 * The value stored in bpf_dtab[n] represent three states:
217 * NULL: device not opened
218 * BPF_DEV_RESERVED: device opening or closing
219 * other: device <n> opened with pointer to storage
220 */
221 #define BPF_DEV_RESERVED ((struct bpf_d *)(uintptr_t)1)
222 static struct bpf_d **bpf_dtab = NULL;
223 static unsigned int bpf_dtab_size = 0;
224 static unsigned int nbpfilter = 0;
225 static unsigned bpf_bpfd_cnt = 0;
226
227 static LCK_GRP_DECLARE(bpf_mlock_grp, "bpf");
228 static LCK_MTX_DECLARE(bpf_mlock_data, &bpf_mlock_grp);
229 static lck_mtx_t *const bpf_mlock = &bpf_mlock_data;
230
231 static int bpf_allocbufs(struct bpf_d *);
232 static errno_t bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
233 static int bpf_detachd(struct bpf_d *d);
234 static void bpf_freed(struct bpf_d *);
235 static int bpf_movein(struct uio *, int,
236 struct mbuf **, struct sockaddr *, int *);
237 static int bpf_setif(struct bpf_d *, ifnet_t ifp, bool, bool, bool);
238 static void bpf_timed_out(void *, void *);
239 static void bpf_wakeup(struct bpf_d *);
240 static uint32_t get_pkt_trunc_len(struct bpf_packet *);
241 static void catchpacket(struct bpf_d *, struct bpf_packet *, u_int, int);
242 static void reset_d(struct bpf_d *);
243 static int bpf_setf(struct bpf_d *, u_int, user_addr_t, u_long);
244 static int bpf_getdltlist(struct bpf_d *, caddr_t, struct proc *);
245 static int bpf_setdlt(struct bpf_d *, u_int);
246 static int bpf_set_traffic_class(struct bpf_d *, int);
247 static void bpf_set_packet_service_class(struct mbuf *, int);
248
249 static void bpf_acquire_d(struct bpf_d *);
250 static void bpf_release_d(struct bpf_d *);
251
252 static int bpf_devsw_installed;
253
254 void bpf_init(void *unused);
255 static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m);
256
257 /*
258 * Darwin differs from BSD here, the following are static
259 * on BSD and not static on Darwin.
260 */
261 d_open_t bpfopen;
262 d_close_t bpfclose;
263 d_read_t bpfread;
264 d_write_t bpfwrite;
265 ioctl_fcn_t bpfioctl;
266 select_fcn_t bpfselect;
267
268 /* Darwin's cdevsw struct differs slightly from BSDs */
269 #define CDEV_MAJOR 23
270 static const struct cdevsw bpf_cdevsw = {
271 .d_open = bpfopen,
272 .d_close = bpfclose,
273 .d_read = bpfread,
274 .d_write = bpfwrite,
275 .d_ioctl = bpfioctl,
276 .d_stop = eno_stop,
277 .d_reset = eno_reset,
278 .d_ttys = NULL,
279 .d_select = bpfselect,
280 .d_mmap = eno_mmap,
281 .d_strategy = eno_strat,
282 .d_reserved_1 = eno_getc,
283 .d_reserved_2 = eno_putc,
284 .d_type = 0
285 };
286
287 #define SOCKADDR_HDR_LEN offsetof(struct sockaddr, sa_data)
288
289 static int
bpf_movein(struct uio * uio,int linktype,struct mbuf ** mp,struct sockaddr * sockp,int * datlen)290 bpf_movein(struct uio *uio, int linktype, struct mbuf **mp,
291 struct sockaddr *sockp, int *datlen)
292 {
293 struct mbuf *m;
294 int error;
295 int len;
296 uint8_t sa_family;
297 int hlen;
298
299 switch (linktype) {
300 #if SLIP
301 case DLT_SLIP:
302 sa_family = AF_INET;
303 hlen = 0;
304 break;
305 #endif /* SLIP */
306
307 case DLT_EN10MB:
308 sa_family = AF_UNSPEC;
309 /* XXX Would MAXLINKHDR be better? */
310 hlen = sizeof(struct ether_header);
311 break;
312
313 #if FDDI
314 case DLT_FDDI:
315 #if defined(__FreeBSD__) || defined(__bsdi__)
316 sa_family = AF_IMPLINK;
317 hlen = 0;
318 #else
319 sa_family = AF_UNSPEC;
320 /* XXX 4(FORMAC)+6(dst)+6(src)+3(LLC)+5(SNAP) */
321 hlen = 24;
322 #endif
323 break;
324 #endif /* FDDI */
325
326 case DLT_RAW:
327 case DLT_NULL:
328 sa_family = AF_UNSPEC;
329 hlen = 0;
330 break;
331
332 #ifdef __FreeBSD__
333 case DLT_ATM_RFC1483:
334 /*
335 * en atm driver requires 4-byte atm pseudo header.
336 * though it isn't standard, vpi:vci needs to be
337 * specified anyway.
338 */
339 sa_family = AF_UNSPEC;
340 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
341 break;
342 #endif
343
344 case DLT_PPP:
345 sa_family = AF_UNSPEC;
346 hlen = 4; /* This should match PPP_HDRLEN */
347 break;
348
349 case DLT_APPLE_IP_OVER_IEEE1394:
350 sa_family = AF_UNSPEC;
351 hlen = sizeof(struct firewire_header);
352 break;
353
354 case DLT_IEEE802_11: /* IEEE 802.11 wireless */
355 sa_family = AF_IEEE80211;
356 hlen = 0;
357 break;
358
359 case DLT_IEEE802_11_RADIO:
360 sa_family = AF_IEEE80211;
361 hlen = 0;
362 break;
363
364 default:
365 return EIO;
366 }
367
368 // LP64todo - fix this!
369 len = (int)uio_resid(uio);
370 if (len < hlen || (unsigned)len > MCLBYTES || len - hlen > MCLBYTES) {
371 return EIO;
372 }
373
374 *datlen = len - hlen;
375
376 if (sockp) {
377 /*
378 * Build a sockaddr based on the data link layer type.
379 * We do this at this level because the ethernet header
380 * is copied directly into the data field of the sockaddr.
381 * In the case of SLIP, there is no header and the packet
382 * is forwarded as is.
383 * Also, we are careful to leave room at the front of the mbuf
384 * for the link level header.
385 */
386 if ((hlen + SOCKADDR_HDR_LEN) > sockp->sa_len) {
387 return EIO;
388 }
389 sockp->sa_family = sa_family;
390 } else {
391 /*
392 * We're directly sending the packet data supplied by
393 * the user; we don't need to make room for the link
394 * header, and don't need the header length value any
395 * more, so set it to 0.
396 */
397 hlen = 0;
398 }
399
400 MGETHDR(m, M_WAIT, MT_DATA);
401 if (m == 0) {
402 return ENOBUFS;
403 }
404 if ((unsigned)len > MHLEN) {
405 MCLGET(m, M_WAIT);
406 if ((m->m_flags & M_EXT) == 0) {
407 error = ENOBUFS;
408 goto bad;
409 }
410 }
411 m->m_pkthdr.len = m->m_len = len;
412 m->m_pkthdr.rcvif = NULL;
413 *mp = m;
414
415 /*
416 * Make room for link header.
417 */
418 if (hlen != 0) {
419 m->m_pkthdr.len -= hlen;
420 m->m_len -= hlen;
421 m->m_data += hlen; /* XXX */
422 error = UIOMOVE((caddr_t)sockp->sa_data, hlen, UIO_WRITE, uio);
423 if (error) {
424 goto bad;
425 }
426 }
427 error = UIOMOVE(mtod(m, caddr_t), len - hlen, UIO_WRITE, uio);
428 if (error) {
429 goto bad;
430 }
431
432 /* Check for multicast destination */
433 switch (linktype) {
434 case DLT_EN10MB: {
435 struct ether_header *eh;
436
437 eh = mtod(m, struct ether_header *);
438 if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
439 if (_ether_cmp(etherbroadcastaddr,
440 eh->ether_dhost) == 0) {
441 m->m_flags |= M_BCAST;
442 } else {
443 m->m_flags |= M_MCAST;
444 }
445 }
446 break;
447 }
448 }
449
450 return 0;
451 bad:
452 m_freem(m);
453 return error;
454 }
455
456 /*
457 * The dynamic addition of a new device node must block all processes that
458 * are opening the last device so that no process will get an unexpected
459 * ENOENT
460 */
461 static void
bpf_make_dev_t(int maj)462 bpf_make_dev_t(int maj)
463 {
464 static int bpf_growing = 0;
465 unsigned int cur_size = nbpfilter, i;
466
467 if (nbpfilter >= bpf_maxdevices) {
468 return;
469 }
470
471 while (bpf_growing) {
472 /* Wait until new device has been created */
473 (void) tsleep((caddr_t)&bpf_growing, PZERO, "bpf_growing", 0);
474 }
475 if (nbpfilter > cur_size) {
476 /* other thread grew it already */
477 return;
478 }
479 bpf_growing = 1;
480
481 /* need to grow bpf_dtab first */
482 if (nbpfilter == bpf_dtab_size) {
483 unsigned int new_dtab_size;
484 struct bpf_d **new_dtab = NULL;
485
486 new_dtab_size = bpf_dtab_size + NBPFILTER;
487 new_dtab = krealloc_type(struct bpf_d *,
488 bpf_dtab_size, new_dtab_size, bpf_dtab, Z_WAITOK | Z_ZERO);
489 if (new_dtab == 0) {
490 os_log_error(OS_LOG_DEFAULT, "bpf_make_dev_t: malloc bpf_dtab failed");
491 goto done;
492 }
493 bpf_dtab = new_dtab;
494 bpf_dtab_size = new_dtab_size;
495 }
496 i = nbpfilter++;
497 (void) devfs_make_node(makedev(maj, i),
498 DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0600,
499 "bpf%d", i);
500 done:
501 bpf_growing = 0;
502 wakeup((caddr_t)&bpf_growing);
503 }
504
505 /*
506 * Attach file to the bpf interface, i.e. make d listen on bp.
507 */
508 static errno_t
bpf_attachd(struct bpf_d * d,struct bpf_if * bp)509 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
510 {
511 int first = bp->bif_dlist == NULL;
512 int error = 0;
513
514 /*
515 * Point d at bp, and add d to the interface's list of listeners.
516 * Finally, point the driver's bpf cookie at the interface so
517 * it will divert packets to bpf.
518 */
519 d->bd_bif = bp;
520 d->bd_next = bp->bif_dlist;
521 bp->bif_dlist = d;
522 bpf_bpfd_cnt++;
523
524 /*
525 * Take a reference on the device even if an error is returned
526 * because we keep the device in the interface's list of listeners
527 */
528 bpf_acquire_d(d);
529
530 if (first) {
531 /* Find the default bpf entry for this ifp */
532 if (bp->bif_ifp->if_bpf == NULL) {
533 struct bpf_if *tmp, *primary = NULL;
534
535 for (tmp = bpf_iflist; tmp; tmp = tmp->bif_next) {
536 if (tmp->bif_ifp == bp->bif_ifp) {
537 primary = tmp;
538 break;
539 }
540 }
541 bp->bif_ifp->if_bpf = primary;
542 }
543 /* Only call dlil_set_bpf_tap for primary dlt */
544 if (bp->bif_ifp->if_bpf == bp) {
545 dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT,
546 bpf_tap_callback);
547 }
548
549 if (bp->bif_tap != NULL) {
550 error = bp->bif_tap(bp->bif_ifp, bp->bif_dlt,
551 BPF_TAP_INPUT_OUTPUT);
552 }
553 }
554
555 /*
556 * Reset the detach flags in case we previously detached an interface
557 */
558 d->bd_flags &= ~(BPF_DETACHING | BPF_DETACHED);
559
560 if (bp->bif_dlt == DLT_PKTAP) {
561 d->bd_flags |= BPF_FINALIZE_PKTAP;
562 } else {
563 d->bd_flags &= ~BPF_FINALIZE_PKTAP;
564 }
565 return error;
566 }
567
568 /*
569 * Detach a file from its interface.
570 *
571 * Return 1 if was closed by some thread, 0 otherwise
572 */
573 static int
bpf_detachd(struct bpf_d * d)574 bpf_detachd(struct bpf_d *d)
575 {
576 struct bpf_d **p;
577 struct bpf_if *bp;
578 struct ifnet *ifp;
579 uint32_t dlt;
580 bpf_tap_func disable_tap;
581 uint8_t bd_promisc;
582
583
584 int bpf_closed = d->bd_flags & BPF_CLOSING;
585 /*
586 * Some other thread already detached
587 */
588 if ((d->bd_flags & (BPF_DETACHED | BPF_DETACHING)) != 0) {
589 goto done;
590 }
591 /*
592 * This thread is doing the detach
593 */
594 d->bd_flags |= BPF_DETACHING;
595
596 ifp = d->bd_bif->bif_ifp;
597 bp = d->bd_bif;
598
599 /* Remove d from the interface's descriptor list. */
600 p = &bp->bif_dlist;
601 while (*p != d) {
602 p = &(*p)->bd_next;
603 if (*p == 0) {
604 panic("bpf_detachd: descriptor not in list");
605 }
606 }
607 *p = (*p)->bd_next;
608 bpf_bpfd_cnt--;
609 disable_tap = NULL;
610 if (bp->bif_dlist == 0) {
611 /*
612 * Let the driver know that there are no more listeners.
613 */
614 /* Only call dlil_set_bpf_tap for primary dlt */
615 if (bp->bif_ifp->if_bpf == bp) {
616 dlil_set_bpf_tap(ifp, BPF_TAP_DISABLE, NULL);
617 }
618
619 disable_tap = bp->bif_tap;
620 if (disable_tap) {
621 dlt = bp->bif_dlt;
622 }
623
624 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
625 if (bp->bif_ifp == ifp && bp->bif_dlist != 0) {
626 break;
627 }
628 }
629 if (bp == NULL) {
630 ifp->if_bpf = NULL;
631 }
632 }
633 d->bd_bif = NULL;
634 /*
635 * Check if this descriptor had requested promiscuous mode.
636 * If so, turn it off.
637 */
638 bd_promisc = d->bd_promisc;
639 d->bd_promisc = 0;
640
641 lck_mtx_unlock(bpf_mlock);
642 if (bd_promisc) {
643 if (ifnet_set_promiscuous(ifp, 0)) {
644 /*
645 * Something is really wrong if we were able to put
646 * the driver into promiscuous mode, but can't
647 * take it out.
648 * Most likely the network interface is gone.
649 */
650 os_log_error(OS_LOG_DEFAULT,
651 "%s: bpf%d ifnet_set_promiscuous %s failed",
652 __func__, d->bd_dev_minor, if_name(ifp));
653 }
654 }
655
656 if (disable_tap) {
657 disable_tap(ifp, dlt, BPF_TAP_DISABLE);
658 }
659 lck_mtx_lock(bpf_mlock);
660
661 /*
662 * Wake up other thread that are waiting for this thread to finish
663 * detaching
664 */
665 d->bd_flags &= ~BPF_DETACHING;
666 d->bd_flags |= BPF_DETACHED;
667
668 /* Refresh the local variable as d could have been modified */
669 bpf_closed = d->bd_flags & BPF_CLOSING;
670
671 os_log(OS_LOG_DEFAULT, "bpf%d%s detached from %s fcount %llu dcount %llu",
672 d->bd_dev_minor, bpf_closed ? " closed and" : "", if_name(ifp),
673 d->bd_fcount, d->bd_dcount);
674
675 /*
676 * Note that We've kept the reference because we may have dropped
677 * the lock when turning off promiscuous mode
678 */
679 bpf_release_d(d);
680 done:
681 /*
682 * Let the caller know the bpf_d is closed
683 */
684 if (bpf_closed) {
685 return 1;
686 } else {
687 return 0;
688 }
689 }
690
691 /*
692 * Start asynchronous timer, if necessary.
693 * Must be called with bpf_mlock held.
694 */
695 static void
bpf_start_timer(struct bpf_d * d)696 bpf_start_timer(struct bpf_d *d)
697 {
698 uint64_t deadline;
699 struct timeval tv;
700
701 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
702 tv.tv_sec = d->bd_rtout / hz;
703 tv.tv_usec = (d->bd_rtout % hz) * tick;
704
705 clock_interval_to_deadline(
706 (uint32_t)tv.tv_sec * USEC_PER_SEC + tv.tv_usec,
707 NSEC_PER_USEC, &deadline);
708 /*
709 * The state is BPF_IDLE, so the timer hasn't
710 * been started yet, and hasn't gone off yet;
711 * there is no thread call scheduled, so this
712 * won't change the schedule.
713 *
714 * XXX - what if, by the time it gets entered,
715 * the deadline has already passed?
716 */
717 thread_call_enter_delayed(d->bd_thread_call, deadline);
718 d->bd_state = BPF_WAITING;
719 }
720 }
721
722 /*
723 * Cancel asynchronous timer.
724 * Must be called with bpf_mlock held.
725 */
726 static boolean_t
bpf_stop_timer(struct bpf_d * d)727 bpf_stop_timer(struct bpf_d *d)
728 {
729 /*
730 * If the timer has already gone off, this does nothing.
731 * Our caller is expected to set d->bd_state to BPF_IDLE,
732 * with the bpf_mlock, after we are called. bpf_timed_out()
733 * also grabs bpf_mlock, so, if the timer has gone off and
734 * bpf_timed_out() hasn't finished, it's waiting for the
735 * lock; when this thread releases the lock, it will
736 * find the state is BPF_IDLE, and just release the
737 * lock and return.
738 */
739 return thread_call_cancel(d->bd_thread_call);
740 }
741
742 void
bpf_acquire_d(struct bpf_d * d)743 bpf_acquire_d(struct bpf_d *d)
744 {
745 void *lr_saved = __builtin_return_address(0);
746
747 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
748
749 d->bd_refcnt += 1;
750
751 d->bd_ref_lr[d->bd_next_ref_lr] = lr_saved;
752 d->bd_next_ref_lr = (d->bd_next_ref_lr + 1) % BPF_REF_HIST;
753 }
754
755 void
bpf_release_d(struct bpf_d * d)756 bpf_release_d(struct bpf_d *d)
757 {
758 void *lr_saved = __builtin_return_address(0);
759
760 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
761
762 if (d->bd_refcnt <= 0) {
763 panic("%s: %p refcnt <= 0", __func__, d);
764 }
765
766 d->bd_refcnt -= 1;
767
768 d->bd_unref_lr[d->bd_next_unref_lr] = lr_saved;
769 d->bd_next_unref_lr = (d->bd_next_unref_lr + 1) % BPF_REF_HIST;
770
771 if (d->bd_refcnt == 0) {
772 /* Assert the device is detached */
773 if ((d->bd_flags & BPF_DETACHED) == 0) {
774 panic("%s: %p BPF_DETACHED not set", __func__, d);
775 }
776
777 kfree_type(struct bpf_d, d);
778 }
779 }
780
781 /*
782 * Open ethernet device. Returns ENXIO for illegal minor device number,
783 * EBUSY if file is open by another process.
784 */
785 /* ARGSUSED */
786 int
bpfopen(dev_t dev,int flags,__unused int fmt,struct proc * p)787 bpfopen(dev_t dev, int flags, __unused int fmt,
788 struct proc *p)
789 {
790 struct bpf_d *d;
791
792 lck_mtx_lock(bpf_mlock);
793 if ((unsigned int) minor(dev) >= nbpfilter) {
794 lck_mtx_unlock(bpf_mlock);
795 return ENXIO;
796 }
797 /*
798 * New device nodes are created on demand when opening the last one.
799 * The programming model is for processes to loop on the minor starting
800 * at 0 as long as EBUSY is returned. The loop stops when either the
801 * open succeeds or an error other that EBUSY is returned. That means
802 * that bpf_make_dev_t() must block all processes that are opening the
803 * last node. If not all processes are blocked, they could unexpectedly
804 * get ENOENT and abort their opening loop.
805 */
806 if ((unsigned int) minor(dev) == (nbpfilter - 1)) {
807 bpf_make_dev_t(major(dev));
808 }
809
810 /*
811 * Each minor can be opened by only one process. If the requested
812 * minor is in use, return EBUSY.
813 *
814 * Important: bpfopen() and bpfclose() have to check and set the status
815 * of a device in the same lockin context otherwise the device may be
816 * leaked because the vnode use count will be unpextectly greater than 1
817 * when close() is called.
818 */
819 if (bpf_dtab[minor(dev)] == NULL) {
820 /* Reserve while opening */
821 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED;
822 } else {
823 lck_mtx_unlock(bpf_mlock);
824 return EBUSY;
825 }
826 d = kalloc_type(struct bpf_d, Z_WAITOK | Z_ZERO);
827 if (d == NULL) {
828 /* this really is a catastrophic failure */
829 os_log_error(OS_LOG_DEFAULT,
830 "bpfopen: bpf%d kalloc_type bpf_d failed", minor(dev));
831 bpf_dtab[minor(dev)] = NULL;
832 lck_mtx_unlock(bpf_mlock);
833 return ENOMEM;
834 }
835
836 /* Mark "in use" and do most initialization. */
837 bpf_acquire_d(d);
838 d->bd_bufsize = bpf_bufsize;
839 d->bd_sig = SIGIO;
840 d->bd_seesent = 1;
841 d->bd_oflags = flags;
842 d->bd_state = BPF_IDLE;
843 d->bd_traffic_class = SO_TC_BE;
844 d->bd_flags |= BPF_DETACHED;
845 if (bpf_wantpktap) {
846 d->bd_flags |= BPF_WANT_PKTAP;
847 } else {
848 d->bd_flags &= ~BPF_WANT_PKTAP;
849 }
850
851 d->bd_thread_call = thread_call_allocate(bpf_timed_out, d);
852 if (d->bd_thread_call == NULL) {
853 os_log_error(OS_LOG_DEFAULT, "bpfopen: bpf%d malloc thread call failed",
854 minor(dev));
855 bpf_dtab[minor(dev)] = NULL;
856 bpf_release_d(d);
857 lck_mtx_unlock(bpf_mlock);
858
859 return ENOMEM;
860 }
861 d->bd_opened_by = p;
862 uuid_generate(d->bd_uuid);
863 d->bd_pid = proc_pid(p);
864
865 d->bd_dev_minor = minor(dev);
866 bpf_dtab[minor(dev)] = d; /* Mark opened */
867 lck_mtx_unlock(bpf_mlock);
868
869 if (bpf_debug) {
870 os_log(OS_LOG_DEFAULT, "bpf%u opened by %s.%u",
871 d->bd_dev_minor, proc_name_address(p), d->bd_pid);
872 }
873 return 0;
874 }
875
876 /*
877 * Close the descriptor by detaching it from its interface,
878 * deallocating its buffers, and marking it free.
879 */
880 /* ARGSUSED */
881 int
bpfclose(dev_t dev,__unused int flags,__unused int fmt,__unused struct proc * p)882 bpfclose(dev_t dev, __unused int flags, __unused int fmt,
883 __unused struct proc *p)
884 {
885 struct bpf_d *d;
886
887 /* Take BPF lock to ensure no other thread is using the device */
888 lck_mtx_lock(bpf_mlock);
889
890 d = bpf_dtab[minor(dev)];
891 if (d == NULL || d == BPF_DEV_RESERVED) {
892 lck_mtx_unlock(bpf_mlock);
893 return ENXIO;
894 }
895
896 /*
897 * Other threads may call bpd_detachd() if we drop the bpf_mlock
898 */
899 d->bd_flags |= BPF_CLOSING;
900
901 if (bpf_debug != 0) {
902 os_log(OS_LOG_DEFAULT, "%s: bpf%d",
903 __func__, d->bd_dev_minor);
904 }
905
906 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED; /* Reserve while closing */
907
908 /*
909 * Deal with any in-progress timeouts.
910 */
911 switch (d->bd_state) {
912 case BPF_IDLE:
913 /*
914 * Not waiting for a timeout, and no timeout happened.
915 */
916 break;
917
918 case BPF_WAITING:
919 /*
920 * Waiting for a timeout.
921 * Cancel any timer that has yet to go off,
922 * and mark the state as "closing".
923 * Then drop the lock to allow any timers that
924 * *have* gone off to run to completion, and wait
925 * for them to finish.
926 */
927 if (!bpf_stop_timer(d)) {
928 /*
929 * There was no pending call, so the call must
930 * have been in progress. Wait for the call to
931 * complete; we have to drop the lock while
932 * waiting. to let the in-progrss call complete
933 */
934 d->bd_state = BPF_DRAINING;
935 while (d->bd_state == BPF_DRAINING) {
936 msleep((caddr_t)d, bpf_mlock, PRINET,
937 "bpfdraining", NULL);
938 }
939 }
940 d->bd_state = BPF_IDLE;
941 break;
942
943 case BPF_TIMED_OUT:
944 /*
945 * Timer went off, and the timeout routine finished.
946 */
947 d->bd_state = BPF_IDLE;
948 break;
949
950 case BPF_DRAINING:
951 /*
952 * Another thread is blocked on a close waiting for
953 * a timeout to finish.
954 * This "shouldn't happen", as the first thread to enter
955 * bpfclose() will set bpf_dtab[minor(dev)] to 1, and
956 * all subsequent threads should see that and fail with
957 * ENXIO.
958 */
959 panic("Two threads blocked in a BPF close");
960 break;
961 }
962
963 if (d->bd_bif) {
964 bpf_detachd(d);
965 }
966 selthreadclear(&d->bd_sel);
967 thread_call_free(d->bd_thread_call);
968
969 while (d->bd_hbuf_read != 0) {
970 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
971 }
972
973 if (bpf_debug) {
974 os_log(OS_LOG_DEFAULT,
975 "bpf%u closed by %s.%u dcount %llu fcount %llu ccount %llu",
976 d->bd_dev_minor, proc_name_address(p), d->bd_pid,
977 d->bd_dcount, d->bd_fcount, d->bd_bcs.bcs_count_compressed_prefix);
978 }
979
980 bpf_freed(d);
981
982 /* Mark free in same context as bpfopen comes to check */
983 bpf_dtab[minor(dev)] = NULL; /* Mark closed */
984
985 bpf_release_d(d);
986
987 lck_mtx_unlock(bpf_mlock);
988
989 return 0;
990 }
991
992 #define BPF_SLEEP bpf_sleep
993
994 static int
bpf_sleep(struct bpf_d * d,int pri,const char * wmesg,int timo)995 bpf_sleep(struct bpf_d *d, int pri, const char *wmesg, int timo)
996 {
997 u_int64_t abstime = 0;
998
999 if (timo != 0) {
1000 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
1001 }
1002
1003 return msleep1((caddr_t)d, bpf_mlock, pri, wmesg, abstime);
1004 }
1005
1006 static void
bpf_finalize_pktap(struct bpf_hdr * hp,struct pktap_header * pktaphdr)1007 bpf_finalize_pktap(struct bpf_hdr *hp, struct pktap_header *pktaphdr)
1008 {
1009 if (pktaphdr->pth_flags & PTH_FLAG_V2_HDR) {
1010 struct pktap_v2_hdr *pktap_v2_hdr;
1011
1012 pktap_v2_hdr = (struct pktap_v2_hdr *)pktaphdr;
1013
1014 if (pktap_v2_hdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1015 pktap_v2_finalize_proc_info(pktap_v2_hdr);
1016 }
1017 } else {
1018 if (pktaphdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1019 pktap_finalize_proc_info(pktaphdr);
1020 }
1021
1022 if (pktaphdr->pth_flags & PTH_FLAG_TSTAMP) {
1023 hp->bh_tstamp.tv_sec = pktaphdr->pth_tstamp.tv_sec;
1024 hp->bh_tstamp.tv_usec = pktaphdr->pth_tstamp.tv_usec;
1025 }
1026 }
1027 }
1028
1029 /*
1030 * Rotate the packet buffers in descriptor d. Move the store buffer
1031 * into the hold slot, and the free buffer into the store slot.
1032 * Zero the length of the new store buffer.
1033 *
1034 * Note: in head drop mode, the hold buffer can be dropped so the fist packet of the
1035 * store buffer cannot be compressed as it otherwise would refer to deleted data
1036 * in a dropped hold buffer that the reader process does know about
1037 */
1038 #define ROTATE_BUFFERS(d) do { \
1039 if (d->bd_hbuf_read != 0) \
1040 panic("rotating bpf buffers during read"); \
1041 (d)->bd_hbuf = (d)->bd_sbuf; \
1042 (d)->bd_hlen = (d)->bd_slen; \
1043 (d)->bd_hcnt = (d)->bd_scnt; \
1044 (d)->bd_sbuf = (d)->bd_fbuf; \
1045 (d)->bd_slen = 0; \
1046 (d)->bd_scnt = 0; \
1047 (d)->bd_fbuf = NULL; \
1048 if ((d)->bd_headdrop != 0) \
1049 (d)->bd_prev_slen = 0; \
1050 } while(false)
1051
1052 /*
1053 * bpfread - read next chunk of packets from buffers
1054 */
1055 int
bpfread(dev_t dev,struct uio * uio,int ioflag)1056 bpfread(dev_t dev, struct uio *uio, int ioflag)
1057 {
1058 struct bpf_d *d;
1059 caddr_t hbuf;
1060 int timed_out, hbuf_len;
1061 int error;
1062 int flags;
1063
1064 lck_mtx_lock(bpf_mlock);
1065
1066 d = bpf_dtab[minor(dev)];
1067 if (d == NULL || d == BPF_DEV_RESERVED ||
1068 (d->bd_flags & BPF_CLOSING) != 0) {
1069 lck_mtx_unlock(bpf_mlock);
1070 return ENXIO;
1071 }
1072
1073 bpf_acquire_d(d);
1074
1075 /*
1076 * Restrict application to use a buffer the same size as
1077 * as kernel buffers.
1078 */
1079 if (uio_resid(uio) != d->bd_bufsize) {
1080 bpf_release_d(d);
1081 lck_mtx_unlock(bpf_mlock);
1082 return EINVAL;
1083 }
1084
1085 if (d->bd_state == BPF_WAITING) {
1086 bpf_stop_timer(d);
1087 }
1088
1089 timed_out = (d->bd_state == BPF_TIMED_OUT);
1090 d->bd_state = BPF_IDLE;
1091
1092 while (d->bd_hbuf_read != 0) {
1093 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
1094 }
1095
1096 if ((d->bd_flags & BPF_CLOSING) != 0) {
1097 bpf_release_d(d);
1098 lck_mtx_unlock(bpf_mlock);
1099 return ENXIO;
1100 }
1101 /*
1102 * If the hold buffer is empty, then do a timed sleep, which
1103 * ends when the timeout expires or when enough packets
1104 * have arrived to fill the store buffer.
1105 */
1106 while (d->bd_hbuf == 0) {
1107 if ((d->bd_immediate || timed_out || (ioflag & IO_NDELAY)) &&
1108 d->bd_slen != 0) {
1109 /*
1110 * We're in immediate mode, or are reading
1111 * in non-blocking mode, or a timer was
1112 * started before the read (e.g., by select()
1113 * or poll()) and has expired and a packet(s)
1114 * either arrived since the previous
1115 * read or arrived while we were asleep.
1116 * Rotate the buffers and return what's here.
1117 */
1118 ROTATE_BUFFERS(d);
1119 break;
1120 }
1121
1122 /*
1123 * No data is available, check to see if the bpf device
1124 * is still pointed at a real interface. If not, return
1125 * ENXIO so that the userland process knows to rebind
1126 * it before using it again.
1127 */
1128 if (d->bd_bif == NULL) {
1129 bpf_release_d(d);
1130 lck_mtx_unlock(bpf_mlock);
1131 return ENXIO;
1132 }
1133 if (ioflag & IO_NDELAY) {
1134 bpf_release_d(d);
1135 lck_mtx_unlock(bpf_mlock);
1136 return EWOULDBLOCK;
1137 }
1138 error = BPF_SLEEP(d, PRINET | PCATCH, "bpf", d->bd_rtout);
1139 /*
1140 * Make sure device is still opened
1141 */
1142 if ((d->bd_flags & BPF_CLOSING) != 0) {
1143 bpf_release_d(d);
1144 lck_mtx_unlock(bpf_mlock);
1145 return ENXIO;
1146 }
1147
1148 while (d->bd_hbuf_read != 0) {
1149 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading",
1150 NULL);
1151 }
1152
1153 if ((d->bd_flags & BPF_CLOSING) != 0) {
1154 bpf_release_d(d);
1155 lck_mtx_unlock(bpf_mlock);
1156 return ENXIO;
1157 }
1158
1159 if (error == EINTR || error == ERESTART) {
1160 if (d->bd_hbuf != NULL) {
1161 /*
1162 * Because we msleep, the hold buffer might
1163 * be filled when we wake up. Avoid rotating
1164 * in this case.
1165 */
1166 break;
1167 }
1168 if (d->bd_slen != 0) {
1169 /*
1170 * Sometimes we may be interrupted often and
1171 * the sleep above will not timeout.
1172 * Regardless, we should rotate the buffers
1173 * if there's any new data pending and
1174 * return it.
1175 */
1176 ROTATE_BUFFERS(d);
1177 break;
1178 }
1179 bpf_release_d(d);
1180 lck_mtx_unlock(bpf_mlock);
1181 if (error == ERESTART) {
1182 os_log(OS_LOG_DEFAULT, "%s: bpf%d ERESTART to EINTR",
1183 __func__, d->bd_dev_minor);
1184 error = EINTR;
1185 }
1186 return error;
1187 }
1188 if (error == EWOULDBLOCK) {
1189 /*
1190 * On a timeout, return what's in the buffer,
1191 * which may be nothing. If there is something
1192 * in the store buffer, we can rotate the buffers.
1193 */
1194 if (d->bd_hbuf) {
1195 /*
1196 * We filled up the buffer in between
1197 * getting the timeout and arriving
1198 * here, so we don't need to rotate.
1199 */
1200 break;
1201 }
1202
1203 if (d->bd_slen == 0) {
1204 bpf_release_d(d);
1205 lck_mtx_unlock(bpf_mlock);
1206 return 0;
1207 }
1208 ROTATE_BUFFERS(d);
1209 break;
1210 }
1211 }
1212 /*
1213 * At this point, we know we have something in the hold slot.
1214 */
1215
1216 /*
1217 * Set the hold buffer read. So we do not
1218 * rotate the buffers until the hold buffer
1219 * read is complete. Also to avoid issues resulting
1220 * from page faults during disk sleep (<rdar://problem/13436396>).
1221 */
1222 d->bd_hbuf_read = 1;
1223 hbuf = d->bd_hbuf;
1224 hbuf_len = d->bd_hlen;
1225 flags = d->bd_flags;
1226 d->bd_bcs.bcs_total_read += d->bd_hcnt;
1227 lck_mtx_unlock(bpf_mlock);
1228
1229 /*
1230 * Before we move data to userland, we fill out the extended
1231 * header fields.
1232 */
1233 if (flags & BPF_EXTENDED_HDR) {
1234 char *p;
1235
1236 p = hbuf;
1237 while (p < hbuf + hbuf_len) {
1238 struct bpf_hdr_ext *ehp;
1239 uint32_t flowid;
1240 struct so_procinfo soprocinfo;
1241 int found = 0;
1242
1243 ehp = (struct bpf_hdr_ext *)(void *)p;
1244 if ((flowid = ehp->bh_flowid) != 0) {
1245 if (ehp->bh_flags & BPF_HDR_EXT_FLAGS_TCP) {
1246 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_TCP;
1247 found = inp_findinpcb_procinfo(&tcbinfo,
1248 flowid, &soprocinfo);
1249 } else if (ehp->bh_flags == BPF_HDR_EXT_FLAGS_UDP) {
1250 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_UDP;
1251 found = inp_findinpcb_procinfo(&udbinfo,
1252 flowid, &soprocinfo);
1253 }
1254 if (found == 1) {
1255 ehp->bh_pid = soprocinfo.spi_pid;
1256 strlcpy(&ehp->bh_comm[0], &soprocinfo.spi_proc_name[0], sizeof(ehp->bh_comm));
1257 }
1258 ehp->bh_flowid = 0;
1259 }
1260
1261 if ((flags & BPF_FINALIZE_PKTAP) != 0 && ehp->bh_complen == 0) {
1262 struct pktap_header *pktaphdr;
1263
1264 pktaphdr = (struct pktap_header *)(void *)
1265 (p + BPF_WORDALIGN(ehp->bh_hdrlen));
1266
1267 bpf_finalize_pktap((struct bpf_hdr *) ehp,
1268 pktaphdr);
1269 }
1270 p += BPF_WORDALIGN(ehp->bh_hdrlen + ehp->bh_caplen);
1271 }
1272 } else if (flags & BPF_FINALIZE_PKTAP) {
1273 char *p;
1274
1275 p = hbuf;
1276
1277 while (p < hbuf + hbuf_len) {
1278 struct bpf_hdr *hp;
1279 struct pktap_header *pktaphdr;
1280
1281 hp = (struct bpf_hdr *)(void *)p;
1282
1283 /*
1284 * Cannot finalize a compressed pktap header as we may not have
1285 * all the fields present
1286 */
1287 if (d->bd_flags & BPF_COMP_ENABLED) {
1288 struct bpf_comp_hdr *hcp;
1289
1290 hcp = (struct bpf_comp_hdr *)(void *)p;
1291
1292 if (hcp->bh_complen != 0) {
1293 p += BPF_WORDALIGN(hcp->bh_hdrlen + hcp->bh_caplen);
1294 continue;
1295 }
1296 }
1297
1298 pktaphdr = (struct pktap_header *)(void *)
1299 (p + BPF_WORDALIGN(hp->bh_hdrlen));
1300
1301 bpf_finalize_pktap(hp, pktaphdr);
1302
1303 p += BPF_WORDALIGN(hp->bh_hdrlen + hp->bh_caplen);
1304 }
1305 }
1306
1307 /*
1308 * Move data from hold buffer into user space.
1309 * We know the entire buffer is transferred since
1310 * we checked above that the read buffer is bpf_bufsize bytes.
1311 */
1312 error = UIOMOVE(hbuf, hbuf_len, UIO_READ, uio);
1313
1314 lck_mtx_lock(bpf_mlock);
1315 /*
1316 * Make sure device is still opened
1317 */
1318 if ((d->bd_flags & BPF_CLOSING) != 0) {
1319 bpf_release_d(d);
1320 lck_mtx_unlock(bpf_mlock);
1321 return ENXIO;
1322 }
1323
1324 d->bd_hbuf_read = 0;
1325 d->bd_fbuf = d->bd_hbuf;
1326 d->bd_hbuf = NULL;
1327 d->bd_hlen = 0;
1328 d->bd_hcnt = 0;
1329 wakeup((caddr_t)d);
1330
1331 bpf_release_d(d);
1332 lck_mtx_unlock(bpf_mlock);
1333 return error;
1334 }
1335
1336 /*
1337 * If there are processes sleeping on this descriptor, wake them up.
1338 */
1339 static void
bpf_wakeup(struct bpf_d * d)1340 bpf_wakeup(struct bpf_d *d)
1341 {
1342 if (d->bd_state == BPF_WAITING) {
1343 bpf_stop_timer(d);
1344 d->bd_state = BPF_IDLE;
1345 }
1346 wakeup((caddr_t)d);
1347 if (d->bd_async && d->bd_sig && d->bd_sigio) {
1348 pgsigio(d->bd_sigio, d->bd_sig);
1349 }
1350
1351 selwakeup(&d->bd_sel);
1352 if ((d->bd_flags & BPF_KNOTE)) {
1353 KNOTE(&d->bd_sel.si_note, 1);
1354 }
1355 }
1356
1357 static void
bpf_timed_out(void * arg,__unused void * dummy)1358 bpf_timed_out(void *arg, __unused void *dummy)
1359 {
1360 struct bpf_d *d = (struct bpf_d *)arg;
1361
1362 lck_mtx_lock(bpf_mlock);
1363 if (d->bd_state == BPF_WAITING) {
1364 /*
1365 * There's a select or kqueue waiting for this; if there's
1366 * now stuff to read, wake it up.
1367 */
1368 d->bd_state = BPF_TIMED_OUT;
1369 if (d->bd_slen != 0) {
1370 bpf_wakeup(d);
1371 }
1372 } else if (d->bd_state == BPF_DRAINING) {
1373 /*
1374 * A close is waiting for this to finish.
1375 * Mark it as finished, and wake the close up.
1376 */
1377 d->bd_state = BPF_IDLE;
1378 bpf_wakeup(d);
1379 }
1380 lck_mtx_unlock(bpf_mlock);
1381 }
1382
1383 /* keep in sync with bpf_movein above: */
1384 #define MAX_DATALINK_HDR_LEN (sizeof(struct firewire_header))
1385
1386 int
bpfwrite(dev_t dev,struct uio * uio,__unused int ioflag)1387 bpfwrite(dev_t dev, struct uio *uio, __unused int ioflag)
1388 {
1389 struct bpf_d *d;
1390 struct ifnet *ifp;
1391 struct mbuf *m = NULL;
1392 int error;
1393 char dst_buf[SOCKADDR_HDR_LEN + MAX_DATALINK_HDR_LEN];
1394 int datlen = 0;
1395 int bif_dlt;
1396 int bd_hdrcmplt;
1397
1398 lck_mtx_lock(bpf_mlock);
1399
1400 d = bpf_dtab[minor(dev)];
1401 if (d == NULL || d == BPF_DEV_RESERVED ||
1402 (d->bd_flags & BPF_CLOSING) != 0) {
1403 lck_mtx_unlock(bpf_mlock);
1404 return ENXIO;
1405 }
1406
1407 bpf_acquire_d(d);
1408
1409 ++d->bd_wcount;
1410
1411 if (d->bd_bif == 0) {
1412 ++d->bd_wdcount;
1413 bpf_release_d(d);
1414 lck_mtx_unlock(bpf_mlock);
1415 return ENXIO;
1416 }
1417
1418 ifp = d->bd_bif->bif_ifp;
1419
1420 if (IFNET_IS_MANAGEMENT(ifp) &&
1421 IOCurrentTaskHasEntitlement(MANAGEMENT_DATA_ENTITLEMENT) == false) {
1422 ++d->bd_wdcount;
1423 bpf_release_d(d);
1424 lck_mtx_unlock(bpf_mlock);
1425 return ENETDOWN;
1426 }
1427
1428 if ((ifp->if_flags & IFF_UP) == 0) {
1429 ++d->bd_wdcount;
1430 bpf_release_d(d);
1431 lck_mtx_unlock(bpf_mlock);
1432 return ENETDOWN;
1433 }
1434 if (uio_resid(uio) == 0) {
1435 bpf_release_d(d);
1436 lck_mtx_unlock(bpf_mlock);
1437 return 0;
1438 }
1439 ((struct sockaddr *)dst_buf)->sa_len = sizeof(dst_buf);
1440
1441 /*
1442 * fix for PR-6849527
1443 * geting variables onto stack before dropping lock for bpf_movein()
1444 */
1445 bif_dlt = (int)d->bd_bif->bif_dlt;
1446 bd_hdrcmplt = d->bd_hdrcmplt;
1447
1448 /* bpf_movein allocating mbufs; drop lock */
1449 lck_mtx_unlock(bpf_mlock);
1450
1451 error = bpf_movein(uio, bif_dlt, &m,
1452 bd_hdrcmplt ? NULL : (struct sockaddr *)dst_buf,
1453 &datlen);
1454
1455 /* take the lock again */
1456 lck_mtx_lock(bpf_mlock);
1457 if (error != 0) {
1458 ++d->bd_wdcount;
1459 bpf_release_d(d);
1460 lck_mtx_unlock(bpf_mlock);
1461 return error;
1462 }
1463
1464 /* verify the device is still open */
1465 if ((d->bd_flags & BPF_CLOSING) != 0) {
1466 ++d->bd_wdcount;
1467 bpf_release_d(d);
1468 lck_mtx_unlock(bpf_mlock);
1469 m_freem(m);
1470 return ENXIO;
1471 }
1472
1473 if (d->bd_bif == NULL) {
1474 ++d->bd_wdcount;
1475 bpf_release_d(d);
1476 lck_mtx_unlock(bpf_mlock);
1477 m_free(m);
1478 return ENXIO;
1479 }
1480
1481 if ((unsigned)datlen > ifp->if_mtu) {
1482 ++d->bd_wdcount;
1483 bpf_release_d(d);
1484 lck_mtx_unlock(bpf_mlock);
1485 m_freem(m);
1486 return EMSGSIZE;
1487 }
1488
1489 bpf_set_packet_service_class(m, d->bd_traffic_class);
1490
1491 lck_mtx_unlock(bpf_mlock);
1492
1493 /*
1494 * The driver frees the mbuf.
1495 */
1496 if (d->bd_hdrcmplt) {
1497 if (d->bd_bif->bif_send) {
1498 error = d->bd_bif->bif_send(ifp, d->bd_bif->bif_dlt, m);
1499 } else {
1500 error = dlil_output(ifp, 0, m, NULL, NULL, 1, NULL);
1501 }
1502 } else {
1503 error = dlil_output(ifp, PF_INET, m, NULL,
1504 (struct sockaddr *)dst_buf, 0, NULL);
1505 }
1506
1507 lck_mtx_lock(bpf_mlock);
1508 if (error != 0) {
1509 ++d->bd_wdcount;
1510 }
1511 bpf_release_d(d);
1512 lck_mtx_unlock(bpf_mlock);
1513
1514 return error;
1515 }
1516
1517 /*
1518 * Reset a descriptor by flushing its packet buffer and clearing the
1519 * receive and drop counts.
1520 */
1521 static void
reset_d(struct bpf_d * d)1522 reset_d(struct bpf_d *d)
1523 {
1524 if (d->bd_hbuf_read != 0) {
1525 panic("resetting buffers during read");
1526 }
1527
1528 if (d->bd_hbuf) {
1529 /* Free the hold buffer. */
1530 d->bd_fbuf = d->bd_hbuf;
1531 d->bd_hbuf = NULL;
1532 }
1533 d->bd_slen = 0;
1534 d->bd_hlen = 0;
1535 d->bd_scnt = 0;
1536 d->bd_hcnt = 0;
1537 d->bd_rcount = 0;
1538 d->bd_dcount = 0;
1539 d->bd_fcount = 0;
1540 d->bd_wcount = 0;
1541 d->bd_wdcount = 0;
1542
1543 d->bd_prev_slen = 0;
1544 }
1545
1546 static struct bpf_d *
bpf_get_device_from_uuid(uuid_t uuid)1547 bpf_get_device_from_uuid(uuid_t uuid)
1548 {
1549 unsigned int i;
1550
1551 for (i = 0; i < nbpfilter; i++) {
1552 struct bpf_d *d = bpf_dtab[i];
1553
1554 if (d == NULL || d == BPF_DEV_RESERVED ||
1555 (d->bd_flags & BPF_CLOSING) != 0) {
1556 continue;
1557 }
1558 if (uuid_compare(uuid, d->bd_uuid) == 0) {
1559 return d;
1560 }
1561 }
1562
1563 return NULL;
1564 }
1565
1566 /*
1567 * The BIOCSETUP command "atomically" attach to the interface and
1568 * copy the buffer from another interface. This minimizes the risk
1569 * of missing packet because this is done while holding
1570 * the BPF global lock
1571 */
1572 static int
bpf_setup(struct bpf_d * d_to,uuid_t uuid_from,ifnet_t ifp)1573 bpf_setup(struct bpf_d *d_to, uuid_t uuid_from, ifnet_t ifp)
1574 {
1575 struct bpf_d *d_from;
1576 int error = 0;
1577
1578 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
1579
1580 /*
1581 * Sanity checks
1582 */
1583 d_from = bpf_get_device_from_uuid(uuid_from);
1584 if (d_from == NULL) {
1585 error = ENOENT;
1586 os_log_error(OS_LOG_DEFAULT,
1587 "%s: uuids not found error %d",
1588 __func__, error);
1589 return error;
1590 }
1591 if (d_from->bd_opened_by != d_to->bd_opened_by) {
1592 error = EACCES;
1593 os_log_error(OS_LOG_DEFAULT,
1594 "%s: processes not matching error %d",
1595 __func__, error);
1596 return error;
1597 }
1598
1599 /*
1600 * Prevent any read while copying
1601 */
1602 while (d_to->bd_hbuf_read != 0) {
1603 msleep((caddr_t)d_to, bpf_mlock, PRINET, __func__, NULL);
1604 }
1605 d_to->bd_hbuf_read = 1;
1606
1607 while (d_from->bd_hbuf_read != 0) {
1608 msleep((caddr_t)d_from, bpf_mlock, PRINET, __func__, NULL);
1609 }
1610 d_from->bd_hbuf_read = 1;
1611
1612 /*
1613 * Verify the devices have not been closed
1614 */
1615 if (d_to->bd_flags & BPF_CLOSING) {
1616 error = ENXIO;
1617 os_log_error(OS_LOG_DEFAULT,
1618 "%s: d_to is closing error %d",
1619 __func__, error);
1620 goto done;
1621 }
1622 if (d_from->bd_flags & BPF_CLOSING) {
1623 error = ENXIO;
1624 os_log_error(OS_LOG_DEFAULT,
1625 "%s: d_from is closing error %d",
1626 __func__, error);
1627 goto done;
1628 }
1629
1630 /*
1631 * For now require the same buffer size
1632 */
1633 if (d_from->bd_bufsize != d_to->bd_bufsize) {
1634 error = EINVAL;
1635 os_log_error(OS_LOG_DEFAULT,
1636 "%s: bufsizes not matching error %d",
1637 __func__, error);
1638 goto done;
1639 }
1640
1641 /*
1642 * Copy relevant options and flags
1643 */
1644 d_to->bd_flags = d_from->bd_flags & (BPF_EXTENDED_HDR | BPF_WANT_PKTAP |
1645 BPF_FINALIZE_PKTAP | BPF_TRUNCATE | BPF_PKTHDRV2 |
1646 BPF_COMP_REQ | BPF_COMP_ENABLED);
1647
1648 d_to->bd_headdrop = d_from->bd_headdrop;
1649
1650 /*
1651 * Allocate and copy the buffers
1652 */
1653 error = bpf_allocbufs(d_to);
1654 if (error != 0) {
1655 goto done;
1656 }
1657
1658 /*
1659 * Make sure the buffers are setup as expected by bpf_setif()
1660 */
1661 ASSERT(d_to->bd_hbuf == NULL);
1662 ASSERT(d_to->bd_sbuf != NULL);
1663 ASSERT(d_to->bd_fbuf != NULL);
1664
1665 /*
1666 * Copy the buffers and update the pointers and counts
1667 */
1668 memcpy(d_to->bd_sbuf, d_from->bd_sbuf, d_from->bd_slen);
1669 d_to->bd_slen = d_from->bd_slen;
1670 d_to->bd_scnt = d_from->bd_scnt;
1671
1672 if (d_from->bd_hbuf != NULL) {
1673 d_to->bd_hbuf = d_to->bd_fbuf;
1674 d_to->bd_fbuf = NULL;
1675 memcpy(d_to->bd_hbuf, d_from->bd_hbuf, d_from->bd_hlen);
1676 }
1677 d_to->bd_hlen = d_from->bd_hlen;
1678 d_to->bd_hcnt = d_from->bd_hcnt;
1679
1680 if (d_to->bd_flags & BPF_COMP_REQ) {
1681 ASSERT(d_to->bd_prev_sbuf != NULL);
1682 ASSERT(d_to->bd_prev_fbuf != NULL);
1683
1684 d_to->bd_prev_slen = d_from->bd_prev_slen;
1685 ASSERT(d_to->bd_prev_slen <= BPF_HDR_COMP_LEN_MAX);
1686 memcpy(d_to->bd_prev_sbuf, d_from->bd_prev_sbuf, BPF_HDR_COMP_LEN_MAX);
1687 }
1688
1689 d_to->bd_bcs = d_from->bd_bcs;
1690
1691 /*
1692 * Attach to the interface:
1693 * - don't reset the buffers
1694 * - we already prevent reads
1695 * - the buffers are already allocated
1696 */
1697 error = bpf_setif(d_to, ifp, false, true, true);
1698 if (error != 0) {
1699 os_log_error(OS_LOG_DEFAULT,
1700 "%s: bpf_setif() failed error %d",
1701 __func__, error);
1702 goto done;
1703 }
1704 done:
1705 d_from->bd_hbuf_read = 0;
1706 wakeup((caddr_t)d_from);
1707
1708 d_to->bd_hbuf_read = 0;
1709 wakeup((caddr_t)d_to);
1710
1711 return error;
1712 }
1713
1714 #if DEVELOPMENT || DEBUG
1715 #define BPF_IOC_LIST \
1716 X(FIONREAD) \
1717 X(SIOCGIFADDR) \
1718 X(BIOCGBLEN) \
1719 X(BIOCSBLEN) \
1720 X(BIOCSETF32) \
1721 X(BIOCSETFNR32) \
1722 X(BIOCSETF64) \
1723 X(BIOCSETFNR64) \
1724 X(BIOCFLUSH) \
1725 X(BIOCPROMISC) \
1726 X(BIOCGDLT) \
1727 X(BIOCGDLTLIST) \
1728 X(BIOCSDLT) \
1729 X(BIOCGETIF) \
1730 X(BIOCSETIF) \
1731 X(BIOCSRTIMEOUT32) \
1732 X(BIOCSRTIMEOUT64) \
1733 X(BIOCGRTIMEOUT32) \
1734 X(BIOCGRTIMEOUT64) \
1735 X(BIOCGSTATS) \
1736 X(BIOCIMMEDIATE) \
1737 X(BIOCVERSION) \
1738 X(BIOCGHDRCMPLT) \
1739 X(BIOCSHDRCMPLT) \
1740 X(BIOCGSEESENT) \
1741 X(BIOCSSEESENT) \
1742 X(BIOCSETTC) \
1743 X(BIOCGETTC) \
1744 X(FIONBIO) \
1745 X(FIOASYNC) \
1746 X(BIOCSRSIG) \
1747 X(BIOCGRSIG) \
1748 X(BIOCSEXTHDR) \
1749 X(BIOCGIFATTACHCOUNT) \
1750 X(BIOCGWANTPKTAP) \
1751 X(BIOCSWANTPKTAP) \
1752 X(BIOCSHEADDROP) \
1753 X(BIOCGHEADDROP) \
1754 X(BIOCSTRUNCATE) \
1755 X(BIOCGETUUID) \
1756 X(BIOCSETUP) \
1757 X(BIOCSPKTHDRV2) \
1758 X(BIOCGHDRCOMP) \
1759 X(BIOCSHDRCOMP) \
1760 X(BIOCGHDRCOMPSTATS) \
1761 X(BIOCGHDRCOMPON)
1762
1763 static void
log_bpf_ioctl_str(struct bpf_d * d,u_long cmd)1764 log_bpf_ioctl_str(struct bpf_d *d, u_long cmd)
1765 {
1766 const char *p = NULL;
1767 char str[32];
1768
1769 #define X(x) case x: { p = #x ; printf("%s\n", p); break; }
1770 switch (cmd) {
1771 BPF_IOC_LIST
1772 }
1773 #undef X
1774 if (p == NULL) {
1775 snprintf(str, sizeof(str), "0x%08x", (unsigned int)cmd);
1776 p = str;
1777 }
1778 os_log(OS_LOG_DEFAULT, "bpfioctl bpf%u %s",
1779 d->bd_dev_minor, p);
1780 }
1781 #endif /* DEVELOPMENT || DEBUG */
1782
1783 /*
1784 * FIONREAD Check for read packet available.
1785 * SIOCGIFADDR Get interface address - convenient hook to driver.
1786 * BIOCGBLEN Get buffer len [for read()].
1787 * BIOCSETF Set ethernet read filter.
1788 * BIOCFLUSH Flush read packet buffer.
1789 * BIOCPROMISC Put interface into promiscuous mode.
1790 * BIOCGDLT Get link layer type.
1791 * BIOCGETIF Get interface name.
1792 * BIOCSETIF Set interface.
1793 * BIOCSRTIMEOUT Set read timeout.
1794 * BIOCGRTIMEOUT Get read timeout.
1795 * BIOCGSTATS Get packet stats.
1796 * BIOCIMMEDIATE Set immediate mode.
1797 * BIOCVERSION Get filter language version.
1798 * BIOCGHDRCMPLT Get "header already complete" flag
1799 * BIOCSHDRCMPLT Set "header already complete" flag
1800 * BIOCGSEESENT Get "see packets sent" flag
1801 * BIOCSSEESENT Set "see packets sent" flag
1802 * BIOCSETTC Set traffic class.
1803 * BIOCGETTC Get traffic class.
1804 * BIOCSEXTHDR Set "extended header" flag
1805 * BIOCSHEADDROP Drop head of the buffer if user is not reading
1806 * BIOCGHEADDROP Get "head-drop" flag
1807 */
1808 /* ARGSUSED */
1809 int
bpfioctl(dev_t dev,u_long cmd,caddr_t addr,__unused int flags,struct proc * p)1810 bpfioctl(dev_t dev, u_long cmd, caddr_t addr, __unused int flags,
1811 struct proc *p)
1812 {
1813 struct bpf_d *d;
1814 int error = 0;
1815 u_int int_arg;
1816 struct ifreq ifr = {};
1817
1818 lck_mtx_lock(bpf_mlock);
1819
1820 d = bpf_dtab[minor(dev)];
1821 if (d == NULL || d == BPF_DEV_RESERVED ||
1822 (d->bd_flags & BPF_CLOSING) != 0) {
1823 lck_mtx_unlock(bpf_mlock);
1824 return ENXIO;
1825 }
1826
1827 bpf_acquire_d(d);
1828
1829 if (d->bd_state == BPF_WAITING) {
1830 bpf_stop_timer(d);
1831 }
1832 d->bd_state = BPF_IDLE;
1833
1834 #if DEVELOPMENT || DEBUG
1835 if (bpf_debug > 0) {
1836 log_bpf_ioctl_str(d, cmd);
1837 }
1838 #endif /* DEVELOPMENT || DEBUG */
1839
1840 switch (cmd) {
1841 default:
1842 error = EINVAL;
1843 break;
1844
1845 /*
1846 * Check for read packet available.
1847 */
1848 case FIONREAD: /* int */
1849 {
1850 int n;
1851
1852 n = d->bd_slen;
1853 if (d->bd_hbuf && d->bd_hbuf_read == 0) {
1854 n += d->bd_hlen;
1855 }
1856
1857 bcopy(&n, addr, sizeof(n));
1858 break;
1859 }
1860
1861 case SIOCGIFADDR: /* struct ifreq */
1862 {
1863 struct ifnet *ifp;
1864
1865 if (d->bd_bif == 0) {
1866 error = EINVAL;
1867 } else {
1868 ifp = d->bd_bif->bif_ifp;
1869 error = ifnet_ioctl(ifp, 0, cmd, addr);
1870 }
1871 break;
1872 }
1873
1874 /*
1875 * Get buffer len [for read()].
1876 */
1877 case BIOCGBLEN: /* u_int */
1878 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
1879 break;
1880
1881 /*
1882 * Set buffer length.
1883 */
1884 case BIOCSBLEN: { /* u_int */
1885 u_int size;
1886
1887 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
1888 /*
1889 * Interface already attached, unable to change buffers
1890 */
1891 error = EINVAL;
1892 break;
1893 }
1894 bcopy(addr, &size, sizeof(size));
1895
1896 if (size > BPF_BUFSIZE_CAP) {
1897 d->bd_bufsize = BPF_BUFSIZE_CAP;
1898
1899 os_log_info(OS_LOG_DEFAULT,
1900 "bpf%d BIOCSBLEN capped to %u from %u",
1901 minor(dev), d->bd_bufsize, size);
1902 } else if (size < BPF_MINBUFSIZE) {
1903 d->bd_bufsize = BPF_MINBUFSIZE;
1904
1905 os_log_info(OS_LOG_DEFAULT,
1906 "bpf%d BIOCSBLEN bumped to %u from %u",
1907 minor(dev), d->bd_bufsize, size);
1908 } else {
1909 d->bd_bufsize = size;
1910
1911 os_log_info(OS_LOG_DEFAULT,
1912 "bpf%d BIOCSBLEN %u",
1913 minor(dev), d->bd_bufsize);
1914 }
1915
1916 /* It's a read/write ioctl */
1917 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
1918 break;
1919 }
1920 /*
1921 * Set link layer read filter.
1922 */
1923 case BIOCSETF32:
1924 case BIOCSETFNR32: { /* struct bpf_program32 */
1925 struct bpf_program32 prg32;
1926
1927 bcopy(addr, &prg32, sizeof(prg32));
1928 error = bpf_setf(d, prg32.bf_len,
1929 CAST_USER_ADDR_T(prg32.bf_insns), cmd);
1930 break;
1931 }
1932
1933 case BIOCSETF64:
1934 case BIOCSETFNR64: { /* struct bpf_program64 */
1935 struct bpf_program64 prg64;
1936
1937 bcopy(addr, &prg64, sizeof(prg64));
1938 error = bpf_setf(d, prg64.bf_len, CAST_USER_ADDR_T(prg64.bf_insns), cmd);
1939 break;
1940 }
1941
1942 /*
1943 * Flush read packet buffer.
1944 */
1945 case BIOCFLUSH:
1946 while (d->bd_hbuf_read != 0) {
1947 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading",
1948 NULL);
1949 }
1950 if ((d->bd_flags & BPF_CLOSING) != 0) {
1951 error = ENXIO;
1952 break;
1953 }
1954 reset_d(d);
1955 break;
1956
1957 /*
1958 * Put interface into promiscuous mode.
1959 */
1960 case BIOCPROMISC:
1961 if (d->bd_bif == 0) {
1962 /*
1963 * No interface attached yet.
1964 */
1965 error = EINVAL;
1966 break;
1967 }
1968 if (d->bd_promisc == 0) {
1969 lck_mtx_unlock(bpf_mlock);
1970 error = ifnet_set_promiscuous(d->bd_bif->bif_ifp, 1);
1971 lck_mtx_lock(bpf_mlock);
1972 if (error == 0) {
1973 d->bd_promisc = 1;
1974 }
1975 }
1976 break;
1977
1978 /*
1979 * Get device parameters.
1980 */
1981 case BIOCGDLT: /* u_int */
1982 if (d->bd_bif == 0) {
1983 error = EINVAL;
1984 } else {
1985 bcopy(&d->bd_bif->bif_dlt, addr, sizeof(u_int));
1986 }
1987 break;
1988
1989 /*
1990 * Get a list of supported data link types.
1991 */
1992 case BIOCGDLTLIST: /* struct bpf_dltlist */
1993 if (d->bd_bif == NULL) {
1994 error = EINVAL;
1995 } else {
1996 error = bpf_getdltlist(d, addr, p);
1997 }
1998 break;
1999
2000 /*
2001 * Set data link type.
2002 */
2003 case BIOCSDLT: /* u_int */
2004 if (d->bd_bif == NULL) {
2005 error = EINVAL;
2006 } else {
2007 u_int dlt;
2008
2009 bcopy(addr, &dlt, sizeof(dlt));
2010
2011 if (dlt == DLT_PKTAP &&
2012 !(d->bd_flags & BPF_WANT_PKTAP)) {
2013 dlt = DLT_RAW;
2014 }
2015 error = bpf_setdlt(d, dlt);
2016 }
2017 break;
2018
2019 /*
2020 * Get interface name.
2021 */
2022 case BIOCGETIF: /* struct ifreq */
2023 if (d->bd_bif == 0) {
2024 error = EINVAL;
2025 } else {
2026 struct ifnet *const ifp = d->bd_bif->bif_ifp;
2027
2028 snprintf(((struct ifreq *)(void *)addr)->ifr_name,
2029 sizeof(ifr.ifr_name), "%s", if_name(ifp));
2030 }
2031 break;
2032
2033 /*
2034 * Set interface.
2035 */
2036 case BIOCSETIF: { /* struct ifreq */
2037 ifnet_t ifp;
2038
2039 bcopy(addr, &ifr, sizeof(ifr));
2040 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2041 ifp = ifunit(ifr.ifr_name);
2042 if (ifp == NULL) {
2043 error = ENXIO;
2044 } else {
2045 error = bpf_setif(d, ifp, true, false, false);
2046 }
2047 break;
2048 }
2049
2050 /*
2051 * Set read timeout.
2052 */
2053 case BIOCSRTIMEOUT32: { /* struct user32_timeval */
2054 struct user32_timeval _tv;
2055 struct timeval tv;
2056
2057 bcopy(addr, &_tv, sizeof(_tv));
2058 tv.tv_sec = _tv.tv_sec;
2059 tv.tv_usec = _tv.tv_usec;
2060
2061 /*
2062 * Subtract 1 tick from tvtohz() since this isn't
2063 * a one-shot timer.
2064 */
2065 if ((error = itimerfix(&tv)) == 0) {
2066 d->bd_rtout = tvtohz(&tv) - 1;
2067 }
2068 break;
2069 }
2070
2071 case BIOCSRTIMEOUT64: { /* struct user64_timeval */
2072 struct user64_timeval _tv;
2073 struct timeval tv;
2074
2075 bcopy(addr, &_tv, sizeof(_tv));
2076 tv.tv_sec = (__darwin_time_t)_tv.tv_sec;
2077 tv.tv_usec = _tv.tv_usec;
2078
2079 /*
2080 * Subtract 1 tick from tvtohz() since this isn't
2081 * a one-shot timer.
2082 */
2083 if ((error = itimerfix(&tv)) == 0) {
2084 d->bd_rtout = tvtohz(&tv) - 1;
2085 }
2086 break;
2087 }
2088
2089 /*
2090 * Get read timeout.
2091 */
2092 case BIOCGRTIMEOUT32: { /* struct user32_timeval */
2093 struct user32_timeval tv;
2094
2095 bzero(&tv, sizeof(tv));
2096 tv.tv_sec = d->bd_rtout / hz;
2097 tv.tv_usec = (d->bd_rtout % hz) * tick;
2098 bcopy(&tv, addr, sizeof(tv));
2099 break;
2100 }
2101
2102 case BIOCGRTIMEOUT64: { /* struct user64_timeval */
2103 struct user64_timeval tv;
2104
2105 bzero(&tv, sizeof(tv));
2106 tv.tv_sec = d->bd_rtout / hz;
2107 tv.tv_usec = (d->bd_rtout % hz) * tick;
2108 bcopy(&tv, addr, sizeof(tv));
2109 break;
2110 }
2111
2112 /*
2113 * Get packet stats.
2114 */
2115 case BIOCGSTATS: { /* struct bpf_stat */
2116 struct bpf_stat bs;
2117
2118 bzero(&bs, sizeof(bs));
2119 bs.bs_recv = (u_int)d->bd_rcount;
2120 bs.bs_drop = (u_int)d->bd_dcount;
2121 bcopy(&bs, addr, sizeof(bs));
2122 break;
2123 }
2124
2125 /*
2126 * Set immediate mode.
2127 */
2128 case BIOCIMMEDIATE: /* u_int */
2129 d->bd_immediate = *(u_char *)(void *)addr;
2130 break;
2131
2132 case BIOCVERSION: { /* struct bpf_version */
2133 struct bpf_version bv;
2134
2135 bzero(&bv, sizeof(bv));
2136 bv.bv_major = BPF_MAJOR_VERSION;
2137 bv.bv_minor = BPF_MINOR_VERSION;
2138 bcopy(&bv, addr, sizeof(bv));
2139 break;
2140 }
2141
2142 /*
2143 * Get "header already complete" flag
2144 */
2145 case BIOCGHDRCMPLT: /* u_int */
2146 bcopy(&d->bd_hdrcmplt, addr, sizeof(u_int));
2147 break;
2148
2149 /*
2150 * Set "header already complete" flag
2151 */
2152 case BIOCSHDRCMPLT: /* u_int */
2153 bcopy(addr, &int_arg, sizeof(int_arg));
2154 d->bd_hdrcmplt = int_arg ? 1 : 0;
2155 break;
2156
2157 /*
2158 * Get "see sent packets" flag
2159 */
2160 case BIOCGSEESENT: /* u_int */
2161 bcopy(&d->bd_seesent, addr, sizeof(u_int));
2162 break;
2163
2164 /*
2165 * Set "see sent packets" flag
2166 */
2167 case BIOCSSEESENT: /* u_int */
2168 bcopy(addr, &d->bd_seesent, sizeof(u_int));
2169 break;
2170
2171 /*
2172 * Set traffic service class
2173 */
2174 case BIOCSETTC: { /* int */
2175 int tc;
2176
2177 bcopy(addr, &tc, sizeof(int));
2178 error = bpf_set_traffic_class(d, tc);
2179 break;
2180 }
2181
2182 /*
2183 * Get traffic service class
2184 */
2185 case BIOCGETTC: /* int */
2186 bcopy(&d->bd_traffic_class, addr, sizeof(int));
2187 break;
2188
2189 case FIONBIO: /* Non-blocking I/O; int */
2190 break;
2191
2192 case FIOASYNC: /* Send signal on receive packets; int */
2193 bcopy(addr, &d->bd_async, sizeof(int));
2194 break;
2195
2196 case BIOCSRSIG: { /* Set receive signal; u_int */
2197 u_int sig;
2198
2199 bcopy(addr, &sig, sizeof(u_int));
2200
2201 if (sig >= NSIG) {
2202 error = EINVAL;
2203 } else {
2204 d->bd_sig = sig;
2205 }
2206 break;
2207 }
2208 case BIOCGRSIG: /* u_int */
2209 bcopy(&d->bd_sig, addr, sizeof(u_int));
2210 break;
2211
2212 case BIOCSEXTHDR: /* u_int */
2213 bcopy(addr, &int_arg, sizeof(int_arg));
2214 if (int_arg) {
2215 d->bd_flags |= BPF_EXTENDED_HDR;
2216 } else {
2217 d->bd_flags &= ~BPF_EXTENDED_HDR;
2218 }
2219 break;
2220
2221 case BIOCGIFATTACHCOUNT: { /* struct ifreq */
2222 ifnet_t ifp;
2223 struct bpf_if *bp;
2224
2225 bcopy(addr, &ifr, sizeof(ifr));
2226 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2227 ifp = ifunit(ifr.ifr_name);
2228 if (ifp == NULL) {
2229 error = ENXIO;
2230 break;
2231 }
2232 ifr.ifr_intval = 0;
2233 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2234 struct bpf_d *bpf_d;
2235
2236 if (bp->bif_ifp == NULL || bp->bif_ifp != ifp) {
2237 continue;
2238 }
2239 for (bpf_d = bp->bif_dlist; bpf_d;
2240 bpf_d = bpf_d->bd_next) {
2241 ifr.ifr_intval += 1;
2242 }
2243 }
2244 bcopy(&ifr, addr, sizeof(ifr));
2245 break;
2246 }
2247 case BIOCGWANTPKTAP: /* u_int */
2248 int_arg = d->bd_flags & BPF_WANT_PKTAP ? 1 : 0;
2249 bcopy(&int_arg, addr, sizeof(int_arg));
2250 break;
2251
2252 case BIOCSWANTPKTAP: /* u_int */
2253 bcopy(addr, &int_arg, sizeof(int_arg));
2254 if (int_arg) {
2255 d->bd_flags |= BPF_WANT_PKTAP;
2256 } else {
2257 d->bd_flags &= ~BPF_WANT_PKTAP;
2258 }
2259 break;
2260
2261 case BIOCSHEADDROP:
2262 bcopy(addr, &int_arg, sizeof(int_arg));
2263 d->bd_headdrop = int_arg ? 1 : 0;
2264 break;
2265
2266 case BIOCGHEADDROP:
2267 bcopy(&d->bd_headdrop, addr, sizeof(int));
2268 break;
2269
2270 case BIOCSTRUNCATE:
2271 bcopy(addr, &int_arg, sizeof(int_arg));
2272 if (int_arg) {
2273 d->bd_flags |= BPF_TRUNCATE;
2274 } else {
2275 d->bd_flags &= ~BPF_TRUNCATE;
2276 }
2277 break;
2278
2279 case BIOCGETUUID:
2280 bcopy(&d->bd_uuid, addr, sizeof(uuid_t));
2281 break;
2282
2283 case BIOCSETUP: {
2284 struct bpf_setup_args bsa;
2285 ifnet_t ifp;
2286
2287 bcopy(addr, &bsa, sizeof(struct bpf_setup_args));
2288 bsa.bsa_ifname[IFNAMSIZ - 1] = 0;
2289 ifp = ifunit(bsa.bsa_ifname);
2290 if (ifp == NULL) {
2291 error = ENXIO;
2292 os_log_error(OS_LOG_DEFAULT,
2293 "%s: ifnet not found for %s error %d",
2294 __func__, bsa.bsa_ifname, error);
2295 break;
2296 }
2297
2298 error = bpf_setup(d, bsa.bsa_uuid, ifp);
2299 break;
2300 }
2301 case BIOCSPKTHDRV2:
2302 bcopy(addr, &int_arg, sizeof(int_arg));
2303 if (int_arg != 0) {
2304 d->bd_flags |= BPF_PKTHDRV2;
2305 } else {
2306 d->bd_flags &= ~BPF_PKTHDRV2;
2307 }
2308 break;
2309
2310 case BIOCGPKTHDRV2:
2311 int_arg = d->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
2312 bcopy(&int_arg, addr, sizeof(int_arg));
2313 break;
2314
2315 case BIOCGHDRCOMP:
2316 int_arg = d->bd_flags & BPF_COMP_REQ ? 1 : 0;
2317 bcopy(&int_arg, addr, sizeof(int_arg));
2318 break;
2319
2320 case BIOCSHDRCOMP:
2321 bcopy(addr, &int_arg, sizeof(int_arg));
2322 if (int_arg != 0 && int_arg != 1) {
2323 return EINVAL;
2324 }
2325 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
2326 /*
2327 * Interface already attached, unable to change buffers
2328 */
2329 error = EINVAL;
2330 break;
2331 }
2332 if (int_arg != 0) {
2333 d->bd_flags |= BPF_COMP_REQ;
2334 if (bpf_hdr_comp_enable != 0) {
2335 d->bd_flags |= BPF_COMP_ENABLED;
2336 }
2337 } else {
2338 d->bd_flags &= ~(BPF_COMP_REQ | BPF_COMP_ENABLED);
2339 }
2340 break;
2341
2342 case BIOCGHDRCOMPON:
2343 int_arg = d->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
2344 bcopy(&int_arg, addr, sizeof(int_arg));
2345 break;
2346
2347 case BIOCGHDRCOMPSTATS: {
2348 struct bpf_comp_stats bcs = {};
2349
2350 bcs = d->bd_bcs;
2351
2352 bcopy(&bcs, addr, sizeof(bcs));
2353 break;
2354 }
2355 }
2356
2357 bpf_release_d(d);
2358 lck_mtx_unlock(bpf_mlock);
2359
2360 return error;
2361 }
2362
2363 /*
2364 * Set d's packet filter program to fp. If this file already has a filter,
2365 * free it and replace it. Returns EINVAL for bogus requests.
2366 */
2367 static int
bpf_setf(struct bpf_d * d,u_int bf_len,user_addr_t bf_insns,u_long cmd)2368 bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns,
2369 u_long cmd)
2370 {
2371 struct bpf_insn *fcode, *old;
2372 u_int flen, size;
2373
2374 while (d->bd_hbuf_read != 0) {
2375 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
2376 }
2377
2378 if ((d->bd_flags & BPF_CLOSING) != 0) {
2379 return ENXIO;
2380 }
2381
2382 old = d->bd_filter;
2383 if (bf_insns == USER_ADDR_NULL) {
2384 if (bf_len != 0) {
2385 return EINVAL;
2386 }
2387 d->bd_filter = NULL;
2388 reset_d(d);
2389 if (old != 0) {
2390 kfree_data_addr(old);
2391 }
2392 return 0;
2393 }
2394 flen = bf_len;
2395 if (flen > BPF_MAXINSNS) {
2396 return EINVAL;
2397 }
2398
2399 size = flen * sizeof(struct bpf_insn);
2400 fcode = (struct bpf_insn *) kalloc_data(size, Z_WAITOK | Z_ZERO);
2401 if (fcode == NULL) {
2402 return ENOMEM;
2403 }
2404 if (copyin(bf_insns, (caddr_t)fcode, size) == 0 &&
2405 bpf_validate(fcode, (int)flen)) {
2406 d->bd_filter = fcode;
2407
2408 if (cmd == BIOCSETF32 || cmd == BIOCSETF64) {
2409 reset_d(d);
2410 }
2411
2412 if (old != 0) {
2413 kfree_data_addr(old);
2414 }
2415
2416 return 0;
2417 }
2418 kfree_data(fcode, size);
2419 return EINVAL;
2420 }
2421
2422 /*
2423 * Detach a file from its current interface (if attached at all) and attach
2424 * to the interface indicated by the name stored in ifr.
2425 * Return an errno or 0.
2426 */
2427 static int
bpf_setif(struct bpf_d * d,ifnet_t theywant,bool do_reset,bool has_hbuf_read,bool has_bufs_allocated)2428 bpf_setif(struct bpf_d *d, ifnet_t theywant, bool do_reset, bool has_hbuf_read,
2429 bool has_bufs_allocated)
2430 {
2431 struct bpf_if *bp;
2432 int error;
2433
2434 while (d->bd_hbuf_read != 0 && !has_hbuf_read) {
2435 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
2436 }
2437
2438 if ((d->bd_flags & BPF_CLOSING) != 0) {
2439 return ENXIO;
2440 }
2441
2442 /*
2443 * Look through attached interfaces for the named one.
2444 */
2445 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2446 struct ifnet *ifp = bp->bif_ifp;
2447
2448 if (ifp == 0 || ifp != theywant) {
2449 continue;
2450 }
2451 /*
2452 * Do not use DLT_PKTAP, unless requested explicitly
2453 */
2454 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2455 continue;
2456 }
2457 /*
2458 * Skip the coprocessor interface
2459 */
2460 if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) {
2461 continue;
2462 }
2463 /*
2464 * We found the requested interface.
2465 * Allocate the packet buffers.
2466 */
2467 if (has_bufs_allocated == false) {
2468 error = bpf_allocbufs(d);
2469 if (error != 0) {
2470 return error;
2471 }
2472 }
2473 /*
2474 * Detach if attached to something else.
2475 */
2476 if (bp != d->bd_bif) {
2477 if (d->bd_bif != NULL) {
2478 if (bpf_detachd(d) != 0) {
2479 return ENXIO;
2480 }
2481 }
2482 if (bpf_attachd(d, bp) != 0) {
2483 return ENXIO;
2484 }
2485 }
2486 if (do_reset) {
2487 reset_d(d);
2488 }
2489 os_log(OS_LOG_DEFAULT, "bpf%u attached to %s",
2490 d->bd_dev_minor, if_name(theywant));
2491 return 0;
2492 }
2493 /* Not found. */
2494 return ENXIO;
2495 }
2496
2497 /*
2498 * Get a list of available data link type of the interface.
2499 */
2500 static int
bpf_getdltlist(struct bpf_d * d,caddr_t addr,struct proc * p)2501 bpf_getdltlist(struct bpf_d *d, caddr_t addr, struct proc *p)
2502 {
2503 u_int n;
2504 int error;
2505 struct ifnet *ifp;
2506 struct bpf_if *bp;
2507 user_addr_t dlist;
2508 struct bpf_dltlist bfl;
2509
2510 bcopy(addr, &bfl, sizeof(bfl));
2511 if (proc_is64bit(p)) {
2512 dlist = (user_addr_t)bfl.bfl_u.bflu_pad;
2513 } else {
2514 dlist = CAST_USER_ADDR_T(bfl.bfl_u.bflu_list);
2515 }
2516
2517 ifp = d->bd_bif->bif_ifp;
2518 n = 0;
2519 error = 0;
2520
2521 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2522 if (bp->bif_ifp != ifp) {
2523 continue;
2524 }
2525 /*
2526 * Do not use DLT_PKTAP, unless requested explicitly
2527 */
2528 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2529 continue;
2530 }
2531 if (dlist != USER_ADDR_NULL) {
2532 if (n >= bfl.bfl_len) {
2533 return ENOMEM;
2534 }
2535 error = copyout(&bp->bif_dlt, dlist,
2536 sizeof(bp->bif_dlt));
2537 if (error != 0) {
2538 break;
2539 }
2540 dlist += sizeof(bp->bif_dlt);
2541 }
2542 n++;
2543 }
2544 bfl.bfl_len = n;
2545 bcopy(&bfl, addr, sizeof(bfl));
2546
2547 return error;
2548 }
2549
2550 /*
2551 * Set the data link type of a BPF instance.
2552 */
2553 static int
bpf_setdlt(struct bpf_d * d,uint32_t dlt)2554 bpf_setdlt(struct bpf_d *d, uint32_t dlt)
2555 {
2556 int error, opromisc;
2557 struct ifnet *ifp;
2558 struct bpf_if *bp;
2559
2560 if (d->bd_bif->bif_dlt == dlt) {
2561 return 0;
2562 }
2563
2564 while (d->bd_hbuf_read != 0) {
2565 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
2566 }
2567
2568 if ((d->bd_flags & BPF_CLOSING) != 0) {
2569 return ENXIO;
2570 }
2571
2572 ifp = d->bd_bif->bif_ifp;
2573 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2574 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) {
2575 /*
2576 * Do not use DLT_PKTAP, unless requested explicitly
2577 */
2578 if (bp->bif_dlt == DLT_PKTAP &&
2579 !(d->bd_flags & BPF_WANT_PKTAP)) {
2580 continue;
2581 }
2582 break;
2583 }
2584 }
2585 if (bp != NULL) {
2586 opromisc = d->bd_promisc;
2587 if (bpf_detachd(d) != 0) {
2588 return ENXIO;
2589 }
2590 error = bpf_attachd(d, bp);
2591 if (error != 0) {
2592 os_log_error(OS_LOG_DEFAULT,
2593 "bpf_setdlt: bpf%d bpf_attachd %s error %d",
2594 d->bd_dev_minor, if_name(bp->bif_ifp),
2595 error);
2596 return error;
2597 }
2598 reset_d(d);
2599 if (opromisc) {
2600 lck_mtx_unlock(bpf_mlock);
2601 error = ifnet_set_promiscuous(bp->bif_ifp, 1);
2602 lck_mtx_lock(bpf_mlock);
2603 if (error != 0) {
2604 os_log_error(OS_LOG_DEFAULT,
2605 "bpf_setdlt: bpf%d ifpromisc %s error %d",
2606 d->bd_dev_minor, if_name(bp->bif_ifp), error);
2607 } else {
2608 d->bd_promisc = 1;
2609 }
2610 }
2611 }
2612 return bp == NULL ? EINVAL : 0;
2613 }
2614
2615 static int
bpf_set_traffic_class(struct bpf_d * d,int tc)2616 bpf_set_traffic_class(struct bpf_d *d, int tc)
2617 {
2618 int error = 0;
2619
2620 if (!SO_VALID_TC(tc)) {
2621 error = EINVAL;
2622 } else {
2623 d->bd_traffic_class = tc;
2624 }
2625
2626 return error;
2627 }
2628
2629 static void
bpf_set_packet_service_class(struct mbuf * m,int tc)2630 bpf_set_packet_service_class(struct mbuf *m, int tc)
2631 {
2632 if (!(m->m_flags & M_PKTHDR)) {
2633 return;
2634 }
2635
2636 VERIFY(SO_VALID_TC(tc));
2637 (void) m_set_service_class(m, so_tc2msc(tc));
2638 }
2639
2640 /*
2641 * Support for select()
2642 *
2643 * Return true iff the specific operation will not block indefinitely.
2644 * Otherwise, return false but make a note that a selwakeup() must be done.
2645 */
2646 int
bpfselect(dev_t dev,int which,void * wql,struct proc * p)2647 bpfselect(dev_t dev, int which, void * wql, struct proc *p)
2648 {
2649 struct bpf_d *d;
2650 int ret = 0;
2651
2652 lck_mtx_lock(bpf_mlock);
2653
2654 d = bpf_dtab[minor(dev)];
2655 if (d == NULL || d == BPF_DEV_RESERVED ||
2656 (d->bd_flags & BPF_CLOSING) != 0) {
2657 lck_mtx_unlock(bpf_mlock);
2658 return ENXIO;
2659 }
2660
2661 bpf_acquire_d(d);
2662
2663 if (d->bd_bif == NULL) {
2664 bpf_release_d(d);
2665 lck_mtx_unlock(bpf_mlock);
2666 return ENXIO;
2667 }
2668
2669 while (d->bd_hbuf_read != 0) {
2670 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_reading", NULL);
2671 }
2672
2673 if ((d->bd_flags & BPF_CLOSING) != 0) {
2674 bpf_release_d(d);
2675 lck_mtx_unlock(bpf_mlock);
2676 return ENXIO;
2677 }
2678
2679 switch (which) {
2680 case FREAD:
2681 if (d->bd_hlen != 0 ||
2682 ((d->bd_immediate ||
2683 d->bd_state == BPF_TIMED_OUT) && d->bd_slen != 0)) {
2684 ret = 1; /* read has data to return */
2685 } else {
2686 /*
2687 * Read has no data to return.
2688 * Make the select wait, and start a timer if
2689 * necessary.
2690 */
2691 selrecord(p, &d->bd_sel, wql);
2692 bpf_start_timer(d);
2693 }
2694 break;
2695
2696 case FWRITE:
2697 /* can't determine whether a write would block */
2698 ret = 1;
2699 break;
2700 }
2701
2702 bpf_release_d(d);
2703 lck_mtx_unlock(bpf_mlock);
2704
2705 return ret;
2706 }
2707
2708 /*
2709 * Support for kevent() system call. Register EVFILT_READ filters and
2710 * reject all others.
2711 */
2712 int bpfkqfilter(dev_t dev, struct knote *kn);
2713 static void filt_bpfdetach(struct knote *);
2714 static int filt_bpfread(struct knote *, long);
2715 static int filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev);
2716 static int filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev);
2717
2718 SECURITY_READ_ONLY_EARLY(struct filterops) bpfread_filtops = {
2719 .f_isfd = 1,
2720 .f_detach = filt_bpfdetach,
2721 .f_event = filt_bpfread,
2722 .f_touch = filt_bpftouch,
2723 .f_process = filt_bpfprocess,
2724 };
2725
2726 static int
filt_bpfread_common(struct knote * kn,struct kevent_qos_s * kev,struct bpf_d * d)2727 filt_bpfread_common(struct knote *kn, struct kevent_qos_s *kev, struct bpf_d *d)
2728 {
2729 int ready = 0;
2730 int64_t data = 0;
2731
2732 if (d->bd_immediate) {
2733 /*
2734 * If there's data in the hold buffer, it's the
2735 * amount of data a read will return.
2736 *
2737 * If there's no data in the hold buffer, but
2738 * there's data in the store buffer, a read will
2739 * immediately rotate the store buffer to the
2740 * hold buffer, the amount of data in the store
2741 * buffer is the amount of data a read will
2742 * return.
2743 *
2744 * If there's no data in either buffer, we're not
2745 * ready to read.
2746 */
2747 data = (d->bd_hlen == 0 || d->bd_hbuf_read != 0 ?
2748 d->bd_slen : d->bd_hlen);
2749 int64_t lowwat = knote_low_watermark(kn);
2750 if (lowwat > d->bd_bufsize) {
2751 lowwat = d->bd_bufsize;
2752 }
2753 ready = (data >= lowwat);
2754 } else {
2755 /*
2756 * If there's data in the hold buffer, it's the
2757 * amount of data a read will return.
2758 *
2759 * If there's no data in the hold buffer, but
2760 * there's data in the store buffer, if the
2761 * timer has expired a read will immediately
2762 * rotate the store buffer to the hold buffer,
2763 * so the amount of data in the store buffer is
2764 * the amount of data a read will return.
2765 *
2766 * If there's no data in either buffer, or there's
2767 * no data in the hold buffer and the timer hasn't
2768 * expired, we're not ready to read.
2769 */
2770 data = ((d->bd_hlen == 0 || d->bd_hbuf_read != 0) &&
2771 d->bd_state == BPF_TIMED_OUT ? d->bd_slen : d->bd_hlen);
2772 ready = (data > 0);
2773 }
2774 if (!ready) {
2775 bpf_start_timer(d);
2776 } else if (kev) {
2777 knote_fill_kevent(kn, kev, data);
2778 }
2779
2780 return ready;
2781 }
2782
2783 int
bpfkqfilter(dev_t dev,struct knote * kn)2784 bpfkqfilter(dev_t dev, struct knote *kn)
2785 {
2786 struct bpf_d *d;
2787 int res;
2788
2789 /*
2790 * Is this device a bpf?
2791 */
2792 if (major(dev) != CDEV_MAJOR || kn->kn_filter != EVFILT_READ) {
2793 knote_set_error(kn, EINVAL);
2794 return 0;
2795 }
2796
2797 lck_mtx_lock(bpf_mlock);
2798
2799 d = bpf_dtab[minor(dev)];
2800
2801 if (d == NULL || d == BPF_DEV_RESERVED ||
2802 (d->bd_flags & BPF_CLOSING) != 0 ||
2803 d->bd_bif == NULL) {
2804 lck_mtx_unlock(bpf_mlock);
2805 knote_set_error(kn, ENXIO);
2806 return 0;
2807 }
2808
2809 kn->kn_hook = d;
2810 kn->kn_filtid = EVFILTID_BPFREAD;
2811 KNOTE_ATTACH(&d->bd_sel.si_note, kn);
2812 d->bd_flags |= BPF_KNOTE;
2813
2814 /* capture the current state */
2815 res = filt_bpfread_common(kn, NULL, d);
2816
2817 lck_mtx_unlock(bpf_mlock);
2818
2819 return res;
2820 }
2821
2822 static void
filt_bpfdetach(struct knote * kn)2823 filt_bpfdetach(struct knote *kn)
2824 {
2825 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2826
2827 lck_mtx_lock(bpf_mlock);
2828 if (d->bd_flags & BPF_KNOTE) {
2829 KNOTE_DETACH(&d->bd_sel.si_note, kn);
2830 d->bd_flags &= ~BPF_KNOTE;
2831 }
2832 lck_mtx_unlock(bpf_mlock);
2833 }
2834
2835 static int
filt_bpfread(struct knote * kn,long hint)2836 filt_bpfread(struct knote *kn, long hint)
2837 {
2838 #pragma unused(hint)
2839 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2840
2841 return filt_bpfread_common(kn, NULL, d);
2842 }
2843
2844 static int
filt_bpftouch(struct knote * kn,struct kevent_qos_s * kev)2845 filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev)
2846 {
2847 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2848 int res;
2849
2850 lck_mtx_lock(bpf_mlock);
2851
2852 /* save off the lowat threshold and flag */
2853 kn->kn_sdata = kev->data;
2854 kn->kn_sfflags = kev->fflags;
2855
2856 /* output data will be re-generated here */
2857 res = filt_bpfread_common(kn, NULL, d);
2858
2859 lck_mtx_unlock(bpf_mlock);
2860
2861 return res;
2862 }
2863
2864 static int
filt_bpfprocess(struct knote * kn,struct kevent_qos_s * kev)2865 filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev)
2866 {
2867 struct bpf_d *d = (struct bpf_d *)kn->kn_hook;
2868 int res;
2869
2870 lck_mtx_lock(bpf_mlock);
2871 res = filt_bpfread_common(kn, kev, d);
2872 lck_mtx_unlock(bpf_mlock);
2873
2874 return res;
2875 }
2876
2877 /*
2878 * Copy data from an mbuf chain into a buffer. This code is derived
2879 * from m_copydata in kern/uipc_mbuf.c.
2880 */
2881 static void
bpf_mcopy(struct mbuf * m,void * dst_arg,size_t len,size_t offset)2882 bpf_mcopy(struct mbuf *m, void *dst_arg, size_t len, size_t offset)
2883 {
2884 u_int count;
2885 u_char *dst;
2886
2887 dst = dst_arg;
2888
2889 while (offset >= m->m_len) {
2890 offset -= m->m_len;
2891 m = m->m_next;
2892 if (m == NULL) {
2893 panic("bpf_mcopy");
2894 }
2895 continue;
2896 }
2897
2898 while (len > 0) {
2899 if (m == NULL) {
2900 panic("bpf_mcopy");
2901 }
2902 count = MIN(m->m_len - (u_int)offset, (u_int)len);
2903 bcopy((u_char *)mbuf_data(m) + offset, dst, count);
2904 m = m->m_next;
2905 dst += count;
2906 len -= count;
2907 offset = 0;
2908 }
2909 }
2910
2911 static inline void
bpf_tap_imp(ifnet_t ifp,u_int32_t dlt,struct bpf_packet * bpf_pkt,int outbound)2912 bpf_tap_imp(
2913 ifnet_t ifp,
2914 u_int32_t dlt,
2915 struct bpf_packet *bpf_pkt,
2916 int outbound)
2917 {
2918 struct bpf_d *d;
2919 u_int slen;
2920 struct bpf_if *bp;
2921
2922 /*
2923 * It's possible that we get here after the bpf descriptor has been
2924 * detached from the interface; in such a case we simply return.
2925 * Lock ordering is important since we can be called asynchronously
2926 * (from IOKit) to process an inbound packet; when that happens
2927 * we would have been holding its "gateLock" and will be acquiring
2928 * "bpf_mlock" upon entering this routine. Due to that, we release
2929 * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will
2930 * acquire "gateLock" in the IOKit), in order to avoid a deadlock
2931 * when a ifnet_set_promiscuous request simultaneously collides with
2932 * an inbound packet being passed into the tap callback.
2933 */
2934 lck_mtx_lock(bpf_mlock);
2935 if (ifp->if_bpf == NULL) {
2936 lck_mtx_unlock(bpf_mlock);
2937 return;
2938 }
2939 for (bp = ifp->if_bpf; bp != NULL; bp = bp->bif_next) {
2940 if (bp->bif_ifp != ifp) {
2941 /* wrong interface */
2942 bp = NULL;
2943 break;
2944 }
2945 if (dlt == 0 || bp->bif_dlt == dlt) {
2946 /* tapping default DLT or DLT matches */
2947 break;
2948 }
2949 }
2950 if (bp == NULL) {
2951 goto done;
2952 }
2953 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) {
2954 struct bpf_packet *bpf_pkt_saved = bpf_pkt;
2955 struct bpf_packet bpf_pkt_tmp = {};
2956 struct pktap_header_buffer bpfp_header_tmp = {};
2957
2958 if (outbound && !d->bd_seesent) {
2959 continue;
2960 }
2961
2962 ++d->bd_rcount;
2963 slen = bpf_filter(d->bd_filter, (u_char *)bpf_pkt,
2964 (u_int)bpf_pkt->bpfp_total_length, 0);
2965
2966 if (slen != 0) {
2967 if (bp->bif_ifp->if_type == IFT_PKTAP &&
2968 bp->bif_dlt == DLT_PKTAP) {
2969 if (d->bd_flags & BPF_TRUNCATE) {
2970 slen = min(slen, get_pkt_trunc_len(bpf_pkt));
2971 }
2972 /*
2973 * Need to copy the bpf_pkt because the conversion
2974 * to v2 pktap header modifies the content of the
2975 * bpfp_header
2976 */
2977 if ((d->bd_flags & BPF_PKTHDRV2) &&
2978 bpf_pkt->bpfp_header_length <= sizeof(bpfp_header_tmp)) {
2979 bpf_pkt_tmp = *bpf_pkt;
2980
2981 bpf_pkt = &bpf_pkt_tmp;
2982
2983 memcpy(&bpfp_header_tmp, bpf_pkt->bpfp_header,
2984 bpf_pkt->bpfp_header_length);
2985
2986 bpf_pkt->bpfp_header = &bpfp_header_tmp;
2987
2988 convert_to_pktap_header_to_v2(bpf_pkt,
2989 !!(d->bd_flags & BPF_TRUNCATE));
2990 }
2991 }
2992 ++d->bd_fcount;
2993 catchpacket(d, bpf_pkt, slen, outbound);
2994 }
2995 bpf_pkt = bpf_pkt_saved;
2996 }
2997
2998 done:
2999 lck_mtx_unlock(bpf_mlock);
3000 }
3001
3002 static inline void
bpf_tap_mbuf(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * hdr,size_t hlen,int outbound)3003 bpf_tap_mbuf(
3004 ifnet_t ifp,
3005 u_int32_t dlt,
3006 mbuf_t m,
3007 void* hdr,
3008 size_t hlen,
3009 int outbound)
3010 {
3011 struct bpf_packet bpf_pkt;
3012 struct mbuf *m0;
3013
3014 if (ifp->if_bpf == NULL) {
3015 /* quickly check without taking lock */
3016 return;
3017 }
3018 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3019 bpf_pkt.bpfp_mbuf = m;
3020 bpf_pkt.bpfp_total_length = 0;
3021 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
3022 bpf_pkt.bpfp_total_length += m0->m_len;
3023 }
3024 bpf_pkt.bpfp_header = hdr;
3025 if (hdr != NULL) {
3026 bpf_pkt.bpfp_total_length += hlen;
3027 bpf_pkt.bpfp_header_length = hlen;
3028 } else {
3029 bpf_pkt.bpfp_header_length = 0;
3030 }
3031 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3032 }
3033
3034 void
bpf_tap_out(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * hdr,size_t hlen)3035 bpf_tap_out(
3036 ifnet_t ifp,
3037 u_int32_t dlt,
3038 mbuf_t m,
3039 void* hdr,
3040 size_t hlen)
3041 {
3042 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 1);
3043 }
3044
3045 void
bpf_tap_in(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * hdr,size_t hlen)3046 bpf_tap_in(
3047 ifnet_t ifp,
3048 u_int32_t dlt,
3049 mbuf_t m,
3050 void* hdr,
3051 size_t hlen)
3052 {
3053 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 0);
3054 }
3055
3056 /* Callback registered with Ethernet driver. */
3057 static int
bpf_tap_callback(struct ifnet * ifp,struct mbuf * m)3058 bpf_tap_callback(struct ifnet *ifp, struct mbuf *m)
3059 {
3060 bpf_tap_mbuf(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL);
3061
3062 return 0;
3063 }
3064
3065 #if SKYWALK
3066 #include <skywalk/os_skywalk_private.h>
3067
3068 static void
bpf_pktcopy(kern_packet_t pkt,void * dst_arg,size_t len,size_t offset)3069 bpf_pktcopy(kern_packet_t pkt, void *dst_arg, size_t len, size_t offset)
3070 {
3071 kern_buflet_t buflet = NULL;
3072 size_t count;
3073 u_char *dst;
3074
3075 dst = dst_arg;
3076 while (len > 0) {
3077 uint8_t *addr;
3078
3079 u_int32_t buflet_length;
3080
3081 buflet = kern_packet_get_next_buflet(pkt, buflet);
3082 VERIFY(buflet != NULL);
3083 addr = kern_buflet_get_data_address(buflet);
3084 VERIFY(addr != NULL);
3085 addr += kern_buflet_get_data_offset(buflet);
3086 buflet_length = kern_buflet_get_data_length(buflet);
3087 if (offset >= buflet_length) {
3088 offset -= buflet_length;
3089 continue;
3090 }
3091 count = MIN(buflet_length - offset, len);
3092 bcopy((void *)(addr + offset), (void *)dst, count);
3093 dst += count;
3094 len -= count;
3095 offset = 0;
3096 }
3097 }
3098
3099 static inline void
bpf_tap_packet(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * hdr,size_t hlen,int outbound)3100 bpf_tap_packet(
3101 ifnet_t ifp,
3102 u_int32_t dlt,
3103 kern_packet_t pkt,
3104 void* hdr,
3105 size_t hlen,
3106 int outbound)
3107 {
3108 struct bpf_packet bpf_pkt;
3109 struct mbuf * m;
3110
3111 if (ifp->if_bpf == NULL) {
3112 /* quickly check without taking lock */
3113 return;
3114 }
3115 m = kern_packet_get_mbuf(pkt);
3116 if (m != NULL) {
3117 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3118 bpf_pkt.bpfp_mbuf = m;
3119 bpf_pkt.bpfp_total_length = m_length(m);
3120 } else {
3121 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_PKT;
3122 bpf_pkt.bpfp_pkt = pkt;
3123 bpf_pkt.bpfp_total_length = kern_packet_get_data_length(pkt);
3124 }
3125 bpf_pkt.bpfp_header = hdr;
3126 bpf_pkt.bpfp_header_length = hlen;
3127 if (hlen != 0) {
3128 bpf_pkt.bpfp_total_length += hlen;
3129 }
3130 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3131 }
3132
3133 void
bpf_tap_packet_out(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * hdr,size_t hlen)3134 bpf_tap_packet_out(
3135 ifnet_t ifp,
3136 u_int32_t dlt,
3137 kern_packet_t pkt,
3138 void* hdr,
3139 size_t hlen)
3140 {
3141 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 1);
3142 }
3143
3144 void
bpf_tap_packet_in(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * hdr,size_t hlen)3145 bpf_tap_packet_in(
3146 ifnet_t ifp,
3147 u_int32_t dlt,
3148 kern_packet_t pkt,
3149 void* hdr,
3150 size_t hlen)
3151 {
3152 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 0);
3153 }
3154
3155 #endif /* SKYWALK */
3156
3157 static errno_t
bpf_copydata(struct bpf_packet * pkt,size_t off,size_t len,void * out_data)3158 bpf_copydata(struct bpf_packet *pkt, size_t off, size_t len, void* out_data)
3159 {
3160 errno_t err = 0;
3161 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
3162 err = mbuf_copydata(pkt->bpfp_mbuf, off, len, out_data);
3163 #if SKYWALK
3164 } else if (pkt->bpfp_type == BPF_PACKET_TYPE_PKT) {
3165 err = kern_packet_copy_bytes(pkt->bpfp_pkt, off, len, out_data);
3166 #endif /* SKYWALK */
3167 } else {
3168 err = EINVAL;
3169 }
3170
3171 return err;
3172 }
3173
3174 static void
copy_bpf_packet_offset(struct bpf_packet * pkt,void * dst,size_t len,size_t offset)3175 copy_bpf_packet_offset(struct bpf_packet * pkt, void * dst, size_t len, size_t offset)
3176 {
3177 /* copy the optional header */
3178 if (offset < pkt->bpfp_header_length) {
3179 size_t count = MIN(len, pkt->bpfp_header_length - offset);
3180 caddr_t src = (caddr_t)pkt->bpfp_header;
3181 bcopy(src + offset, dst, count);
3182 len -= count;
3183 dst = (void *)((uintptr_t)dst + count);
3184 offset = 0;
3185 } else {
3186 offset -= pkt->bpfp_header_length;
3187 }
3188
3189 if (len == 0) {
3190 /* nothing past the header */
3191 return;
3192 }
3193 /* copy the packet */
3194 switch (pkt->bpfp_type) {
3195 case BPF_PACKET_TYPE_MBUF:
3196 bpf_mcopy(pkt->bpfp_mbuf, dst, len, offset);
3197 break;
3198 #if SKYWALK
3199 case BPF_PACKET_TYPE_PKT:
3200 bpf_pktcopy(pkt->bpfp_pkt, dst, len, offset);
3201 break;
3202 #endif /* SKYWALK */
3203 default:
3204 break;
3205 }
3206 }
3207
3208 static void
copy_bpf_packet(struct bpf_packet * pkt,void * dst,size_t len)3209 copy_bpf_packet(struct bpf_packet * pkt, void * dst, size_t len)
3210 {
3211 copy_bpf_packet_offset(pkt, dst, len, 0);
3212 }
3213
3214 static uint32_t
get_esp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3215 get_esp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3216 const uint32_t remaining_caplen)
3217 {
3218 /*
3219 * For some reason tcpdump expects to have one byte beyond the ESP header
3220 */
3221 uint32_t trunc_len = ESP_HDR_SIZE + 1;
3222
3223 if (trunc_len > remaining_caplen) {
3224 return remaining_caplen;
3225 }
3226
3227 return trunc_len;
3228 }
3229
3230 static uint32_t
get_isakmp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3231 get_isakmp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3232 const uint32_t remaining_caplen)
3233 {
3234 /*
3235 * Include the payload generic header
3236 */
3237 uint32_t trunc_len = ISAKMP_HDR_SIZE;
3238
3239 if (trunc_len > remaining_caplen) {
3240 return remaining_caplen;
3241 }
3242
3243 return trunc_len;
3244 }
3245
3246 static uint32_t
get_isakmp_natt_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3247 get_isakmp_natt_trunc_len(struct bpf_packet *pkt, uint32_t off,
3248 const uint32_t remaining_caplen)
3249 {
3250 int err = 0;
3251 uint32_t trunc_len = 0;
3252 char payload[remaining_caplen];
3253
3254 err = bpf_copydata(pkt, off, remaining_caplen, payload);
3255 if (err != 0) {
3256 return remaining_caplen;
3257 }
3258 /*
3259 * They are three cases:
3260 * - IKE: payload start with 4 bytes header set to zero before ISAKMP header
3261 * - keep alive: 1 byte payload
3262 * - otherwise it's ESP
3263 */
3264 if (remaining_caplen >= 4 &&
3265 payload[0] == 0 && payload[1] == 0 &&
3266 payload[2] == 0 && payload[3] == 0) {
3267 trunc_len = 4 + get_isakmp_trunc_len(pkt, off + 4, remaining_caplen - 4);
3268 } else if (remaining_caplen == 1) {
3269 trunc_len = 1;
3270 } else {
3271 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3272 }
3273
3274 if (trunc_len > remaining_caplen) {
3275 return remaining_caplen;
3276 }
3277
3278 return trunc_len;
3279 }
3280
3281 static uint32_t
get_udp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3282 get_udp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3283 {
3284 int err = 0;
3285 uint32_t trunc_len = sizeof(struct udphdr); /* By default no UDP payload */
3286
3287 if (trunc_len >= remaining_caplen) {
3288 return remaining_caplen;
3289 }
3290
3291 struct udphdr udphdr;
3292 err = bpf_copydata(pkt, off, sizeof(struct udphdr), &udphdr);
3293 if (err != 0) {
3294 return remaining_caplen;
3295 }
3296
3297 u_short sport, dport;
3298
3299 sport = EXTRACT_SHORT(&udphdr.uh_sport);
3300 dport = EXTRACT_SHORT(&udphdr.uh_dport);
3301
3302 if (dport == PORT_DNS || sport == PORT_DNS) {
3303 /*
3304 * Full UDP payload for DNS
3305 */
3306 trunc_len = remaining_caplen;
3307 } else if ((sport == PORT_BOOTPS && dport == PORT_BOOTPC) ||
3308 (sport == PORT_BOOTPC && dport == PORT_BOOTPS)) {
3309 /*
3310 * Full UDP payload for BOOTP and DHCP
3311 */
3312 trunc_len = remaining_caplen;
3313 } else if (dport == PORT_ISAKMP && sport == PORT_ISAKMP) {
3314 /*
3315 * Return the ISAKMP header
3316 */
3317 trunc_len += get_isakmp_trunc_len(pkt, off + sizeof(struct udphdr),
3318 remaining_caplen - sizeof(struct udphdr));
3319 } else if (dport == PORT_ISAKMP_NATT && sport == PORT_ISAKMP_NATT) {
3320 trunc_len += get_isakmp_natt_trunc_len(pkt, off + sizeof(struct udphdr),
3321 remaining_caplen - sizeof(struct udphdr));
3322 }
3323 if (trunc_len >= remaining_caplen) {
3324 return remaining_caplen;
3325 }
3326
3327 return trunc_len;
3328 }
3329
3330 static uint32_t
get_tcp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3331 get_tcp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3332 {
3333 int err = 0;
3334 uint32_t trunc_len = sizeof(struct tcphdr); /* By default no TCP payload */
3335 if (trunc_len >= remaining_caplen) {
3336 return remaining_caplen;
3337 }
3338
3339 struct tcphdr tcphdr;
3340 err = bpf_copydata(pkt, off, sizeof(struct tcphdr), &tcphdr);
3341 if (err != 0) {
3342 return remaining_caplen;
3343 }
3344
3345 u_short sport, dport;
3346 sport = EXTRACT_SHORT(&tcphdr.th_sport);
3347 dport = EXTRACT_SHORT(&tcphdr.th_dport);
3348
3349 if (dport == PORT_DNS || sport == PORT_DNS) {
3350 /*
3351 * Full TCP payload for DNS
3352 */
3353 trunc_len = remaining_caplen;
3354 } else {
3355 trunc_len = (uint16_t)(tcphdr.th_off << 2);
3356 }
3357 if (trunc_len >= remaining_caplen) {
3358 return remaining_caplen;
3359 }
3360
3361 return trunc_len;
3362 }
3363
3364 static uint32_t
get_proto_trunc_len(uint8_t proto,struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3365 get_proto_trunc_len(uint8_t proto, struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3366 {
3367 uint32_t trunc_len;
3368
3369 switch (proto) {
3370 case IPPROTO_ICMP: {
3371 /*
3372 * Full IMCP payload
3373 */
3374 trunc_len = remaining_caplen;
3375 break;
3376 }
3377 case IPPROTO_ICMPV6: {
3378 /*
3379 * Full IMCPV6 payload
3380 */
3381 trunc_len = remaining_caplen;
3382 break;
3383 }
3384 case IPPROTO_IGMP: {
3385 /*
3386 * Full IGMP payload
3387 */
3388 trunc_len = remaining_caplen;
3389 break;
3390 }
3391 case IPPROTO_UDP: {
3392 trunc_len = get_udp_trunc_len(pkt, off, remaining_caplen);
3393 break;
3394 }
3395 case IPPROTO_TCP: {
3396 trunc_len = get_tcp_trunc_len(pkt, off, remaining_caplen);
3397 break;
3398 }
3399 case IPPROTO_ESP: {
3400 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3401 break;
3402 }
3403 default: {
3404 /*
3405 * By default we only include the IP header
3406 */
3407 trunc_len = 0;
3408 break;
3409 }
3410 }
3411 if (trunc_len >= remaining_caplen) {
3412 return remaining_caplen;
3413 }
3414
3415 return trunc_len;
3416 }
3417
3418 static uint32_t
get_ip_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3419 get_ip_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3420 {
3421 int err = 0;
3422 uint32_t iplen = sizeof(struct ip);
3423 if (iplen >= remaining_caplen) {
3424 return remaining_caplen;
3425 }
3426
3427 struct ip iphdr;
3428 err = bpf_copydata(pkt, off, sizeof(struct ip), &iphdr);
3429 if (err != 0) {
3430 return remaining_caplen;
3431 }
3432
3433 uint8_t proto = 0;
3434
3435 iplen = (uint16_t)(iphdr.ip_hl << 2);
3436 if (iplen >= remaining_caplen) {
3437 return remaining_caplen;
3438 }
3439
3440 proto = iphdr.ip_p;
3441 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3442
3443 if (iplen >= remaining_caplen) {
3444 return remaining_caplen;
3445 }
3446
3447 return iplen;
3448 }
3449
3450 static uint32_t
get_ip6_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3451 get_ip6_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3452 {
3453 int err = 0;
3454 uint32_t iplen = sizeof(struct ip6_hdr);
3455 if (iplen >= remaining_caplen) {
3456 return remaining_caplen;
3457 }
3458
3459 struct ip6_hdr ip6hdr;
3460 err = bpf_copydata(pkt, off, sizeof(struct ip6_hdr), &ip6hdr);
3461 if (err != 0) {
3462 return remaining_caplen;
3463 }
3464
3465 uint8_t proto = 0;
3466
3467 /*
3468 * TBD: process the extension headers
3469 */
3470 proto = ip6hdr.ip6_nxt;
3471 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3472
3473 if (iplen >= remaining_caplen) {
3474 return remaining_caplen;
3475 }
3476
3477 return iplen;
3478 }
3479
3480 static uint32_t
get_ether_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3481 get_ether_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3482 {
3483 int err = 0;
3484 uint32_t ethlen = sizeof(struct ether_header);
3485 if (ethlen >= remaining_caplen) {
3486 return remaining_caplen;
3487 }
3488
3489 struct ether_header eh = {};
3490 err = bpf_copydata(pkt, off, sizeof(struct ether_header), &eh);
3491 if (err != 0) {
3492 return remaining_caplen;
3493 }
3494
3495 u_short type = EXTRACT_SHORT(&eh.ether_type);
3496 /* Include full ARP */
3497 if (type == ETHERTYPE_ARP) {
3498 ethlen = remaining_caplen;
3499 } else if (type == ETHERTYPE_IP) {
3500 ethlen += get_ip_trunc_len(pkt, off + sizeof(struct ether_header),
3501 remaining_caplen - ethlen);
3502 } else if (type == ETHERTYPE_IPV6) {
3503 ethlen += get_ip6_trunc_len(pkt, off + sizeof(struct ether_header),
3504 remaining_caplen - ethlen);
3505 } else {
3506 ethlen = MIN(BPF_MIN_PKT_SIZE, remaining_caplen);
3507 }
3508 return ethlen;
3509 }
3510
3511
3512 static uint32_t
get_pkt_trunc_len(struct bpf_packet * pkt)3513 get_pkt_trunc_len(struct bpf_packet *pkt)
3514 {
3515 struct pktap_header *pktap = (struct pktap_header *) (pkt->bpfp_header);
3516 uint32_t in_pkt_len = 0;
3517 uint32_t out_pkt_len = 0;
3518 uint32_t tlen = 0;
3519 uint32_t pre_adjust; // L2 header not in mbuf or kern_packet
3520
3521 // bpfp_total_length must contain the BPF packet header
3522 assert3u(pkt->bpfp_total_length, >=, pkt->bpfp_header_length);
3523
3524 // The BPF packet header must contain the pktap header
3525 assert3u(pkt->bpfp_header_length, >=, pktap->pth_length);
3526
3527 // The pre frame length (L2 header) must be contained in the packet
3528 assert3u(pkt->bpfp_total_length, >=, pktap->pth_length + pktap->pth_frame_pre_length);
3529
3530 /*
3531 * pktap->pth_frame_pre_length is the L2 header length and accounts
3532 * for both L2 header in the packet payload and pre_adjust.
3533 *
3534 * pre_adjust represents an adjustment for a pseudo L2 header that is not
3535 * part of packet payload -- not in the mbuf or kern_packet -- and comes
3536 * just after the pktap header.
3537 *
3538 * pktap->pth_length is the size of the pktap header (exclude pre_adjust)
3539 *
3540 * pkt->bpfp_header_length is (pktap->pth_length + pre_adjust)
3541 */
3542 pre_adjust = (uint32_t)(pkt->bpfp_header_length - pktap->pth_length);
3543
3544 if (pktap->pth_iftype == IFT_ETHER) {
3545 /*
3546 * We need to parse the Ethernet header to find the network layer
3547 * protocol
3548 */
3549 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pre_adjust);
3550
3551 out_pkt_len = get_ether_trunc_len(pkt, 0, in_pkt_len);
3552
3553 tlen = pktap->pth_length + pre_adjust + out_pkt_len;
3554 } else {
3555 /*
3556 * For other interface types, we only know to parse IPv4 and IPv6.
3557 *
3558 * To get to the beginning of the IPv4 or IPv6 packet, we need to to skip
3559 * over the L2 header that is the actual packet payload (mbuf or kern_packet)
3560 */
3561 uint32_t off; // offset past the L2 header in the actual packet payload
3562
3563 off = pktap->pth_frame_pre_length - pre_adjust;
3564
3565 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pktap->pth_frame_pre_length);
3566
3567 if (pktap->pth_protocol_family == AF_INET) {
3568 out_pkt_len = get_ip_trunc_len(pkt, off, in_pkt_len);
3569 } else if (pktap->pth_protocol_family == AF_INET6) {
3570 out_pkt_len = get_ip6_trunc_len(pkt, off, in_pkt_len);
3571 } else {
3572 out_pkt_len = MIN(BPF_MIN_PKT_SIZE, in_pkt_len);
3573 }
3574 tlen = pktap->pth_length + pktap->pth_frame_pre_length + out_pkt_len;
3575 }
3576
3577 // Verify we do not overflow the buffer
3578 if (__improbable(tlen > pkt->bpfp_total_length)) {
3579 bool do_panic = bpf_debug != 0 ? true : false;
3580
3581 #if DEBUG
3582 do_panic = true;
3583 #endif /* DEBUG */
3584 if (do_panic) {
3585 panic("%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
3586 __func__, __LINE__,
3587 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
3588 } else {
3589 os_log(OS_LOG_DEFAULT,
3590 "%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
3591 __func__, __LINE__,
3592 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
3593 }
3594 bpf_trunc_overflow += 1;
3595 tlen = (uint32_t)pkt->bpfp_total_length;
3596 }
3597
3598 return tlen;
3599 }
3600
3601 static uint8_t
get_common_prefix_size(const void * a,const void * b,uint8_t max_bytes)3602 get_common_prefix_size(const void *a, const void *b, uint8_t max_bytes)
3603 {
3604 uint8_t max_words = max_bytes >> 2;
3605 const uint32_t *x = (const uint32_t *)a;
3606 const uint32_t *y = (const uint32_t *)b;
3607 uint8_t i;
3608
3609 for (i = 0; i < max_words; i++) {
3610 if (x[i] != y[i]) {
3611 break;
3612 }
3613 }
3614 return (uint8_t)(i << 2);
3615 }
3616
3617 /*
3618 * Move the packet data from interface memory (pkt) into the
3619 * store buffer. Return 1 if it's time to wakeup a listener (buffer full),
3620 * otherwise 0.
3621 */
3622 static void
catchpacket(struct bpf_d * d,struct bpf_packet * pkt,u_int snaplen,int outbound)3623 catchpacket(struct bpf_d *d, struct bpf_packet * pkt,
3624 u_int snaplen, int outbound)
3625 {
3626 struct bpf_hdr *hp;
3627 struct bpf_hdr_ext *ehp;
3628 uint32_t totlen, curlen;
3629 uint32_t hdrlen, caplen;
3630 int do_wakeup = 0;
3631 u_char *payload;
3632 struct timeval tv;
3633
3634 hdrlen = (d->bd_flags & BPF_EXTENDED_HDR) ? d->bd_bif->bif_exthdrlen :
3635 (d->bd_flags & BPF_COMP_REQ) ? d->bd_bif->bif_comphdrlen:
3636 d->bd_bif->bif_hdrlen;
3637 /*
3638 * Figure out how many bytes to move. If the packet is
3639 * greater or equal to the snapshot length, transfer that
3640 * much. Otherwise, transfer the whole packet (unless
3641 * we hit the buffer size limit).
3642 */
3643 totlen = hdrlen + MIN(snaplen, (int)pkt->bpfp_total_length);
3644 if (totlen > d->bd_bufsize) {
3645 totlen = d->bd_bufsize;
3646 }
3647
3648 if (hdrlen > totlen) {
3649 return;
3650 }
3651
3652 /*
3653 * Round up the end of the previous packet to the next longword.
3654 */
3655 curlen = BPF_WORDALIGN(d->bd_slen);
3656 if (curlen + totlen > d->bd_bufsize) {
3657 /*
3658 * This packet will overflow the storage buffer.
3659 * Rotate the buffers if we can, then wakeup any
3660 * pending reads.
3661 *
3662 * We cannot rotate buffers if a read is in progress
3663 * so drop the packet
3664 */
3665 if (d->bd_hbuf_read != 0) {
3666 ++d->bd_dcount;
3667 return;
3668 }
3669
3670 if (d->bd_fbuf == NULL) {
3671 if (d->bd_headdrop == 0) {
3672 /*
3673 * We haven't completed the previous read yet,
3674 * so drop the packet.
3675 */
3676 ++d->bd_dcount;
3677 return;
3678 }
3679 /*
3680 * Drop the hold buffer as it contains older packets
3681 */
3682 d->bd_dcount += d->bd_hcnt;
3683 d->bd_fbuf = d->bd_hbuf;
3684 ROTATE_BUFFERS(d);
3685 } else {
3686 ROTATE_BUFFERS(d);
3687 }
3688 do_wakeup = 1;
3689 curlen = 0;
3690 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
3691 /*
3692 * Immediate mode is set, or the read timeout has
3693 * already expired during a select call. A packet
3694 * arrived, so the reader should be woken up.
3695 */
3696 do_wakeup = 1;
3697 }
3698
3699 /*
3700 * Append the bpf header.
3701 */
3702 microtime(&tv);
3703 if (d->bd_flags & BPF_EXTENDED_HDR) {
3704 ehp = (struct bpf_hdr_ext *)(void *)(d->bd_sbuf + curlen);
3705 memset(ehp, 0, sizeof(*ehp));
3706 ehp->bh_tstamp.tv_sec = (int)tv.tv_sec;
3707 ehp->bh_tstamp.tv_usec = tv.tv_usec;
3708
3709 ehp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
3710 ehp->bh_hdrlen = (u_short)hdrlen;
3711 caplen = ehp->bh_caplen = totlen - hdrlen;
3712 payload = (u_char *)ehp + hdrlen;
3713
3714 if (outbound) {
3715 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT;
3716 } else {
3717 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN;
3718 }
3719
3720 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
3721 struct mbuf *m = pkt->bpfp_mbuf;
3722
3723 if (outbound) {
3724 /* only do lookups on non-raw INPCB */
3725 if ((m->m_pkthdr.pkt_flags & (PKTF_FLOW_ID |
3726 PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)) ==
3727 (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC) &&
3728 m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
3729 ehp->bh_flowid = m->m_pkthdr.pkt_flowid;
3730 if (m->m_pkthdr.pkt_proto == IPPROTO_TCP) {
3731 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_TCP;
3732 } else if (m->m_pkthdr.pkt_proto == IPPROTO_UDP) {
3733 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_UDP;
3734 }
3735 }
3736 ehp->bh_svc = so_svc2tc(m->m_pkthdr.pkt_svc);
3737 if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) {
3738 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
3739 }
3740 if (m->m_pkthdr.pkt_flags & PKTF_START_SEQ) {
3741 ehp->bh_pktflags |= BPF_PKTFLAGS_START_SEQ;
3742 }
3743 if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT) {
3744 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
3745 }
3746 if (m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA) {
3747 ehp->bh_unsent_bytes =
3748 m->m_pkthdr.bufstatus_if;
3749 ehp->bh_unsent_snd =
3750 m->m_pkthdr.bufstatus_sndbuf;
3751 }
3752 } else {
3753 if (m->m_pkthdr.pkt_flags & PKTF_WAKE_PKT) {
3754 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
3755 }
3756 }
3757 #if SKYWALK
3758 } else {
3759 kern_packet_t kern_pkt = pkt->bpfp_pkt;
3760 packet_flowid_t flowid = 0;
3761
3762 if (outbound) {
3763 /*
3764 * Note: pp_init() asserts that kern_packet_svc_class_t is equivalent
3765 * to mbuf_svc_class_t
3766 */
3767 ehp->bh_svc = so_svc2tc((mbuf_svc_class_t)kern_packet_get_service_class(kern_pkt));
3768 if (kern_packet_get_transport_retransmit(kern_pkt)) {
3769 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
3770 }
3771 if (kern_packet_get_transport_last_packet(kern_pkt)) {
3772 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
3773 }
3774 } else {
3775 if (kern_packet_get_wake_flag(kern_pkt)) {
3776 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
3777 }
3778 }
3779 ehp->bh_trace_tag = kern_packet_get_trace_tag(kern_pkt);
3780 if (kern_packet_get_flowid(kern_pkt, &flowid) == 0) {
3781 ehp->bh_flowid = flowid;
3782 }
3783 #endif /* SKYWALK */
3784 }
3785 } else {
3786 hp = (struct bpf_hdr *)(void *)(d->bd_sbuf + curlen);
3787 memset(hp, 0, BPF_WORDALIGN(sizeof(*hp)));
3788 hp->bh_tstamp.tv_sec = (int)tv.tv_sec;
3789 hp->bh_tstamp.tv_usec = tv.tv_usec;
3790 hp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
3791 hp->bh_hdrlen = (u_short)hdrlen;
3792 caplen = hp->bh_caplen = totlen - hdrlen;
3793 payload = (u_char *)hp + hdrlen;
3794 }
3795 if (d->bd_flags & BPF_COMP_REQ) {
3796 uint8_t common_prefix_size = 0;
3797 uint8_t copy_len = MIN((uint8_t)caplen, BPF_HDR_COMP_LEN_MAX);
3798
3799 copy_bpf_packet(pkt, d->bd_prev_fbuf, copy_len);
3800
3801 if (d->bd_prev_slen != 0) {
3802 common_prefix_size = get_common_prefix_size(d->bd_prev_fbuf,
3803 d->bd_prev_sbuf, MIN(copy_len, d->bd_prev_slen));
3804 }
3805
3806 if (d->bd_flags & BPF_COMP_ENABLED) {
3807 assert3u(caplen, >=, common_prefix_size);
3808 copy_bpf_packet_offset(pkt, payload, caplen - common_prefix_size,
3809 common_prefix_size);
3810 d->bd_slen = curlen + totlen - common_prefix_size;
3811 } else {
3812 copy_bpf_packet(pkt, payload, caplen);
3813 d->bd_slen = curlen + totlen;
3814 }
3815
3816 /*
3817 * Update the caplen only if compression is enabled -- the caller
3818 * must pay attention to bpf_hdr_comp_enable
3819 */
3820 if (d->bd_flags & BPF_EXTENDED_HDR) {
3821 ehp->bh_complen = common_prefix_size;
3822 if (d->bd_flags & BPF_COMP_ENABLED) {
3823 ehp->bh_caplen -= common_prefix_size;
3824 }
3825 } else {
3826 struct bpf_comp_hdr *hcp;
3827
3828 hcp = (struct bpf_comp_hdr *)(void *)(d->bd_sbuf + curlen);
3829 hcp->bh_complen = common_prefix_size;
3830 if (d->bd_flags & BPF_COMP_ENABLED) {
3831 hcp->bh_caplen -= common_prefix_size;
3832 }
3833 }
3834
3835 if (common_prefix_size > 0) {
3836 d->bd_bcs.bcs_total_compressed_prefix_size += common_prefix_size;
3837 if (common_prefix_size > d->bd_bcs.bcs_max_compressed_prefix_size) {
3838 d->bd_bcs.bcs_max_compressed_prefix_size = common_prefix_size;
3839 }
3840 d->bd_bcs.bcs_count_compressed_prefix += 1;
3841 } else {
3842 d->bd_bcs.bcs_count_no_common_prefix += 1;
3843 }
3844
3845 /* The current compression buffer becomes the previous one */
3846 caddr_t tmp = d->bd_prev_sbuf;
3847 d->bd_prev_sbuf = d->bd_prev_fbuf;
3848 d->bd_prev_slen = copy_len;
3849 d->bd_prev_fbuf = tmp;
3850 } else {
3851 /*
3852 * Copy the packet data into the store buffer and update its length.
3853 */
3854 copy_bpf_packet(pkt, payload, caplen);
3855 d->bd_slen = curlen + totlen;
3856 }
3857 d->bd_scnt += 1;
3858 d->bd_bcs.bcs_total_hdr_size += pkt->bpfp_header_length;
3859 d->bd_bcs.bcs_total_size += caplen;
3860
3861 if (do_wakeup) {
3862 bpf_wakeup(d);
3863 }
3864 }
3865
3866
3867 static void
bpf_freebufs(struct bpf_d * d)3868 bpf_freebufs(struct bpf_d *d)
3869 {
3870 if (d->bd_sbuf != NULL) {
3871 kfree_data_addr(d->bd_sbuf);
3872 }
3873 if (d->bd_hbuf != NULL) {
3874 kfree_data_addr(d->bd_hbuf);
3875 }
3876 if (d->bd_fbuf != NULL) {
3877 kfree_data_addr(d->bd_fbuf);
3878 }
3879
3880 if (d->bd_prev_sbuf != NULL) {
3881 kfree_data_addr(d->bd_prev_sbuf);
3882 }
3883 if (d->bd_prev_fbuf != NULL) {
3884 kfree_data_addr(d->bd_prev_fbuf);
3885 }
3886 }
3887 /*
3888 * Initialize all nonzero fields of a descriptor.
3889 */
3890 static int
bpf_allocbufs(struct bpf_d * d)3891 bpf_allocbufs(struct bpf_d *d)
3892 {
3893 bpf_freebufs(d);
3894
3895 d->bd_fbuf = (caddr_t) kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
3896 if (d->bd_fbuf == NULL) {
3897 goto nobufs;
3898 }
3899
3900 d->bd_sbuf = (caddr_t) kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
3901 if (d->bd_sbuf == NULL) {
3902 goto nobufs;
3903 }
3904 d->bd_slen = 0;
3905 d->bd_hlen = 0;
3906 d->bd_scnt = 0;
3907 d->bd_hcnt = 0;
3908
3909 d->bd_prev_slen = 0;
3910 if (d->bd_flags & BPF_COMP_REQ) {
3911 d->bd_prev_sbuf = (caddr_t) kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
3912 if (d->bd_prev_sbuf == NULL) {
3913 goto nobufs;
3914 }
3915 d->bd_prev_fbuf = (caddr_t) kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
3916 if (d->bd_prev_fbuf == NULL) {
3917 goto nobufs;
3918 }
3919 }
3920 return 0;
3921 nobufs:
3922 bpf_freebufs(d);
3923 return ENOMEM;
3924 }
3925
3926 /*
3927 * Free buffers currently in use by a descriptor.
3928 * Called on close.
3929 */
3930 static void
bpf_freed(struct bpf_d * d)3931 bpf_freed(struct bpf_d *d)
3932 {
3933 /*
3934 * We don't need to lock out interrupts since this descriptor has
3935 * been detached from its interface and it yet hasn't been marked
3936 * free.
3937 */
3938 if (d->bd_hbuf_read != 0) {
3939 panic("bpf buffer freed during read");
3940 }
3941
3942 bpf_freebufs(d);
3943
3944 if (d->bd_filter) {
3945 kfree_data_addr(d->bd_filter);
3946 }
3947 }
3948
3949 /*
3950 * Attach an interface to bpf. driverp is a pointer to a (struct bpf_if *)
3951 * in the driver's softc; dlt is the link layer type; hdrlen is the fixed
3952 * size of the link header (variable length headers not yet supported).
3953 */
3954 void
bpfattach(struct ifnet * ifp,u_int dlt,u_int hdrlen)3955 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
3956 {
3957 bpf_attach(ifp, dlt, hdrlen, NULL, NULL);
3958 }
3959
3960 errno_t
bpf_attach(ifnet_t ifp,u_int32_t dlt,u_int32_t hdrlen,bpf_send_func send,bpf_tap_func tap)3961 bpf_attach(
3962 ifnet_t ifp,
3963 u_int32_t dlt,
3964 u_int32_t hdrlen,
3965 bpf_send_func send,
3966 bpf_tap_func tap)
3967 {
3968 struct bpf_if *bp;
3969 struct bpf_if *bp_new;
3970 struct bpf_if *bp_before_first = NULL;
3971 struct bpf_if *bp_first = NULL;
3972 struct bpf_if *bp_last = NULL;
3973 boolean_t found;
3974
3975 /*
3976 * Z_NOFAIL will cause a panic if the allocation fails
3977 */
3978 bp_new = kalloc_type(struct bpf_if, Z_WAITOK | Z_NOFAIL | Z_ZERO);
3979
3980 lck_mtx_lock(bpf_mlock);
3981
3982 /*
3983 * Check if this interface/dlt is already attached. Remember the
3984 * first and last attachment for this interface, as well as the
3985 * element before the first attachment.
3986 */
3987 found = FALSE;
3988 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
3989 if (bp->bif_ifp != ifp) {
3990 if (bp_first != NULL) {
3991 /* no more elements for this interface */
3992 break;
3993 }
3994 bp_before_first = bp;
3995 } else {
3996 if (bp->bif_dlt == dlt) {
3997 found = TRUE;
3998 break;
3999 }
4000 if (bp_first == NULL) {
4001 bp_first = bp;
4002 }
4003 bp_last = bp;
4004 }
4005 }
4006 if (found) {
4007 lck_mtx_unlock(bpf_mlock);
4008 os_log_error(OS_LOG_DEFAULT,
4009 "bpfattach - %s with dlt %d is already attached",
4010 if_name(ifp), dlt);
4011 kfree_type(struct bpf_if, bp_new);
4012 return EEXIST;
4013 }
4014
4015 bp_new->bif_ifp = ifp;
4016 bp_new->bif_dlt = dlt;
4017 bp_new->bif_send = send;
4018 bp_new->bif_tap = tap;
4019
4020 if (bp_first == NULL) {
4021 /* No other entries for this ifp */
4022 bp_new->bif_next = bpf_iflist;
4023 bpf_iflist = bp_new;
4024 } else {
4025 if (ifnet_type(ifp) == IFT_ETHER && dlt == DLT_EN10MB) {
4026 /* Make this the first entry for this interface */
4027 if (bp_before_first != NULL) {
4028 /* point the previous to us */
4029 bp_before_first->bif_next = bp_new;
4030 } else {
4031 /* we're the new head */
4032 bpf_iflist = bp_new;
4033 }
4034 bp_new->bif_next = bp_first;
4035 } else {
4036 /* Add this after the last entry for this interface */
4037 bp_new->bif_next = bp_last->bif_next;
4038 bp_last->bif_next = bp_new;
4039 }
4040 }
4041
4042 /*
4043 * Compute the length of the bpf header. This is not necessarily
4044 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
4045 * that the network layer header begins on a longword boundary (for
4046 * performance reasons and to alleviate alignment restrictions).
4047 */
4048 bp_new->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
4049 bp_new->bif_exthdrlen = BPF_WORDALIGN(hdrlen +
4050 sizeof(struct bpf_hdr_ext)) - hdrlen;
4051 bp_new->bif_comphdrlen = BPF_WORDALIGN(hdrlen +
4052 sizeof(struct bpf_comp_hdr)) - hdrlen;
4053
4054 /* Take a reference on the interface */
4055 ifnet_reference(ifp);
4056
4057 lck_mtx_unlock(bpf_mlock);
4058
4059 return 0;
4060 }
4061
4062 /*
4063 * Detach bpf from an interface. This involves detaching each descriptor
4064 * associated with the interface, and leaving bd_bif NULL. Notify each
4065 * descriptor as it's detached so that any sleepers wake up and get
4066 * ENXIO.
4067 */
4068 void
bpfdetach(struct ifnet * ifp)4069 bpfdetach(struct ifnet *ifp)
4070 {
4071 struct bpf_if *bp, *bp_prev, *bp_next;
4072 struct bpf_d *d;
4073
4074 if (bpf_debug != 0) {
4075 os_log(OS_LOG_DEFAULT, "%s: %s", __func__, if_name(ifp));
4076 }
4077
4078 lck_mtx_lock(bpf_mlock);
4079
4080 /*
4081 * Build the list of devices attached to that interface
4082 * that we need to free while keeping the lock to maintain
4083 * the integrity of the interface list
4084 */
4085 bp_prev = NULL;
4086 for (bp = bpf_iflist; bp != NULL; bp = bp_next) {
4087 bp_next = bp->bif_next;
4088
4089 if (ifp != bp->bif_ifp) {
4090 bp_prev = bp;
4091 continue;
4092 }
4093 /* Unlink from the interface list */
4094 if (bp_prev) {
4095 bp_prev->bif_next = bp->bif_next;
4096 } else {
4097 bpf_iflist = bp->bif_next;
4098 }
4099
4100 /* Detach the devices attached to the interface */
4101 while ((d = bp->bif_dlist) != NULL) {
4102 /*
4103 * Take an extra reference to prevent the device
4104 * from being freed when bpf_detachd() releases
4105 * the reference for the interface list
4106 */
4107 bpf_acquire_d(d);
4108 bpf_detachd(d);
4109 bpf_wakeup(d);
4110 bpf_release_d(d);
4111 }
4112 ifnet_release(ifp);
4113 }
4114
4115 lck_mtx_unlock(bpf_mlock);
4116 }
4117
4118 void
bpf_init(__unused void * unused)4119 bpf_init(__unused void *unused)
4120 {
4121 int maj;
4122
4123 /* bpf_comp_hdr is an overlay of bpf_hdr */
4124 _CASSERT(BPF_WORDALIGN(sizeof(struct bpf_hdr)) ==
4125 BPF_WORDALIGN(sizeof(struct bpf_comp_hdr)));
4126
4127 /* compression length must fits in a byte */
4128 _CASSERT(BPF_HDR_COMP_LEN_MAX <= UCHAR_MAX );
4129
4130 (void) PE_parse_boot_argn("bpf_hdr_comp", &bpf_hdr_comp_enable,
4131 sizeof(bpf_hdr_comp_enable));
4132
4133 if (bpf_devsw_installed == 0) {
4134 bpf_devsw_installed = 1;
4135 maj = cdevsw_add(CDEV_MAJOR, &bpf_cdevsw);
4136 if (maj == -1) {
4137 bpf_devsw_installed = 0;
4138 os_log_error(OS_LOG_DEFAULT,
4139 "bpf_init: failed to allocate a major number");
4140 return;
4141 }
4142
4143 for (int i = 0; i < NBPFILTER; i++) {
4144 bpf_make_dev_t(maj);
4145 }
4146 }
4147 }
4148
4149 static int
4150 sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS
4151 {
4152 #pragma unused(arg1, arg2)
4153 int i, err;
4154
4155 i = bpf_maxbufsize;
4156
4157 err = sysctl_handle_int(oidp, &i, 0, req);
4158 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4159 return err;
4160 }
4161
4162 if (i < 0 || i > BPF_BUFSIZE_CAP) {
4163 i = BPF_BUFSIZE_CAP;
4164 }
4165
4166 bpf_maxbufsize = i;
4167 return err;
4168 }
4169
4170 static int
4171 sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS
4172 {
4173 #pragma unused(arg1, arg2)
4174 int i, err;
4175
4176 i = BPF_BUFSIZE_CAP;
4177
4178 err = sysctl_handle_int(oidp, &i, 0, req);
4179 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4180 return err;
4181 }
4182
4183 return err;
4184 }
4185
4186 /*
4187 * Fill filter statistics
4188 */
4189 static void
bpfstats_fill_xbpf(struct xbpf_d * d,struct bpf_d * bd)4190 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
4191 {
4192 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
4193
4194 d->bd_structsize = sizeof(struct xbpf_d);
4195 d->bd_promisc = bd->bd_promisc != 0 ? 1 : 0;
4196 d->bd_immediate = d->bd_immediate != 0 ? 1 : 0;
4197 d->bd_hdrcmplt = bd->bd_hdrcmplt != 0 ? 1 : 0;
4198 d->bd_async = bd->bd_async != 0 ? 1 : 0;
4199 d->bd_headdrop = bd->bd_headdrop != 0 ? 1 : 0;
4200 d->bd_seesent = bd->bd_seesent != 0 ? 1 : 0;
4201 d->bh_compreq = bd->bd_flags & BPF_COMP_REQ ? 1 : 0;
4202 d->bh_compenabled = bd->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
4203 d->bd_exthdr = bd->bd_flags & BPF_EXTENDED_HDR ? 1 : 0;
4204 d->bd_trunc = bd->bd_flags & BPF_TRUNCATE ? 1 : 0;
4205 d->bd_pkthdrv2 = bd->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
4206
4207 d->bd_dev_minor = (uint8_t)bd->bd_dev_minor;
4208
4209 d->bd_sig = bd->bd_sig;
4210
4211 d->bd_rcount = bd->bd_rcount;
4212 d->bd_dcount = bd->bd_dcount;
4213 d->bd_fcount = bd->bd_fcount;
4214 d->bd_slen = bd->bd_slen;
4215 d->bd_hlen = bd->bd_hlen;
4216 d->bd_bufsize = bd->bd_bufsize;
4217 d->bd_pid = bd->bd_pid;
4218 if (bd->bd_bif != NULL && bd->bd_bif->bif_ifp != NULL) {
4219 strlcpy(d->bd_ifname,
4220 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
4221 }
4222
4223 d->bd_comp_count = bd->bd_bcs.bcs_count_compressed_prefix;
4224 d->bd_comp_size = bd->bd_bcs.bcs_total_compressed_prefix_size;
4225
4226 d->bd_scnt = bd->bd_scnt;
4227 d->bd_hcnt = bd->bd_hcnt;
4228
4229 d->bd_read_count = bd->bd_bcs.bcs_total_read;
4230 d->bd_fsize = bd->bd_bcs.bcs_total_size;
4231 }
4232
4233 /*
4234 * Handle `netstat -B' stats request
4235 */
4236 static int
4237 sysctl_bpf_stats SYSCTL_HANDLER_ARGS
4238 {
4239 int error;
4240 struct xbpf_d *xbdbuf;
4241 unsigned int x_cnt;
4242 vm_size_t buf_size;
4243
4244 if (req->oldptr == USER_ADDR_NULL) {
4245 return SYSCTL_OUT(req, 0, nbpfilter * sizeof(struct xbpf_d));
4246 }
4247 if (nbpfilter == 0) {
4248 return SYSCTL_OUT(req, 0, 0);
4249 }
4250 buf_size = req->oldlen;
4251 xbdbuf = kalloc_data(buf_size, Z_WAITOK | Z_ZERO);
4252
4253 lck_mtx_lock(bpf_mlock);
4254 if (buf_size < (nbpfilter * sizeof(struct xbpf_d))) {
4255 lck_mtx_unlock(bpf_mlock);
4256 kfree_data(xbdbuf, buf_size);
4257 return ENOMEM;
4258 }
4259 x_cnt = 0;
4260 unsigned int i;
4261
4262 for (i = 0; i < nbpfilter; i++) {
4263 struct bpf_d *bd = bpf_dtab[i];
4264 struct xbpf_d *xbd;
4265
4266 if (bd == NULL || bd == BPF_DEV_RESERVED ||
4267 (bd->bd_flags & BPF_CLOSING) != 0) {
4268 continue;
4269 }
4270 VERIFY(x_cnt < nbpfilter);
4271
4272 xbd = &xbdbuf[x_cnt++];
4273 bpfstats_fill_xbpf(xbd, bd);
4274 }
4275 lck_mtx_unlock(bpf_mlock);
4276
4277 error = SYSCTL_OUT(req, xbdbuf, x_cnt * sizeof(struct xbpf_d));
4278 kfree_data(xbdbuf, buf_size);
4279 return error;
4280 }
4281