1 /*
2 * Copyright (c) 2000-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1990, 1991, 1993
30 * The Regents of the University of California. All rights reserved.
31 *
32 * This code is derived from the Stanford/CMU enet packet filter,
33 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
34 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
35 * Berkeley Laboratory.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * 3. All advertising materials mentioning features or use of this software
46 * must display the following acknowledgement:
47 * This product includes software developed by the University of
48 * California, Berkeley and its contributors.
49 * 4. Neither the name of the University nor the names of its contributors
50 * may be used to endorse or promote products derived from this software
51 * without specific prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 *
65 * @(#)bpf.c 8.2 (Berkeley) 3/28/94
66 *
67 * $FreeBSD: src/sys/net/bpf.c,v 1.59.2.5 2001/01/05 04:49:09 jdp Exp $
68 */
69 /*
70 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
71 * support for mandatory and extensible security protections. This notice
72 * is included in support of clause 2.2 (b) of the Apple Public License,
73 * Version 2.0.
74 */
75
76 #include "bpf.h"
77
78 #ifndef __GNUC__
79 #define inline
80 #else
81 #define inline __inline
82 #endif
83
84 #include <sys/param.h>
85 #include <sys/systm.h>
86 #include <sys/conf.h>
87 #include <sys/malloc.h>
88 #include <sys/mbuf.h>
89 #include <sys/time.h>
90 #include <sys/proc.h>
91 #include <sys/signalvar.h>
92 #include <sys/filio.h>
93 #include <sys/sockio.h>
94 #include <sys/ttycom.h>
95 #include <sys/filedesc.h>
96 #include <sys/uio_internal.h>
97 #include <sys/file_internal.h>
98 #include <sys/event.h>
99
100 #include <sys/poll.h>
101
102 #include <sys/socket.h>
103 #include <sys/socketvar.h>
104 #include <sys/vnode.h>
105
106 #include <net/if.h>
107 #include <net/bpf.h>
108 #include <net/bpfdesc.h>
109
110 #include <netinet/in.h>
111 #include <netinet/ip.h>
112 #include <netinet/ip6.h>
113 #include <netinet/in_pcb.h>
114 #include <netinet/in_var.h>
115 #include <netinet/ip_var.h>
116 #include <netinet/tcp.h>
117 #include <netinet/tcp_var.h>
118 #include <netinet/udp.h>
119 #include <netinet/udp_var.h>
120 #include <netinet/if_ether.h>
121 #include <netinet/isakmp.h>
122 #include <netinet6/esp.h>
123 #include <sys/kernel.h>
124 #include <sys/sysctl.h>
125 #include <net/firewire.h>
126
127 #include <miscfs/devfs/devfs.h>
128 #include <net/dlil.h>
129 #include <net/pktap.h>
130
131 #include <kern/assert.h>
132 #include <kern/locks.h>
133 #include <kern/thread_call.h>
134 #include <libkern/section_keywords.h>
135
136 #include <os/log.h>
137
138 #include <IOKit/IOBSD.h>
139
140 #include <net/sockaddr_utils.h>
141
142 extern int tvtohz(struct timeval *);
143 extern const char *proc_name_address(void *);
144
145 #define BPF_BUFSIZE 4096
146
147 #define PRINET 26 /* interruptible */
148
149 #define ISAKMP_HDR_SIZE (sizeof(struct isakmp) + sizeof(struct isakmp_gen))
150 #define ESP_HDR_SIZE sizeof(struct newesp)
151
152 #define BPF_WRITE_LEEWAY 18 /* space for link layer header */
153
154 #define BPF_WRITE_MAX 0x1000000 /* 16 MB arbitrary value */
155
156 typedef void (*pktcopyfunc_t)(const void *, void *, size_t);
157
158 /*
159 * The default read buffer size is patchable.
160 */
161 static unsigned int bpf_bufsize = BPF_BUFSIZE;
162 SYSCTL_INT(_debug, OID_AUTO, bpf_bufsize, CTLFLAG_RW | CTLFLAG_LOCKED,
163 &bpf_bufsize, 0, "");
164
165 __private_extern__ unsigned int bpf_maxbufsize = BPF_MAXBUFSIZE;
166 static int sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS;
167 SYSCTL_PROC(_debug, OID_AUTO, bpf_maxbufsize, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
168 &bpf_maxbufsize, 0,
169 sysctl_bpf_maxbufsize, "I", "Default BPF max buffer size");
170
171 extern const int copysize_limit_panic;
172 #define BPF_BUFSIZE_CAP (copysize_limit_panic >> 1)
173 static int sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS;
174 SYSCTL_PROC(_debug, OID_AUTO, bpf_bufsize_cap, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
175 0, 0,
176 sysctl_bpf_bufsize_cap, "I", "Upper limit on BPF max buffer size");
177
178 #define BPF_MAX_DEVICES 256
179 static unsigned int bpf_maxdevices = BPF_MAX_DEVICES;
180 SYSCTL_UINT(_debug, OID_AUTO, bpf_maxdevices, CTLFLAG_RD | CTLFLAG_LOCKED,
181 &bpf_maxdevices, 0, "");
182
183 /*
184 * bpf_wantpktap controls the defaul visibility of DLT_PKTAP
185 * For OS X is off by default so process need to use the ioctl BPF_WANT_PKTAP
186 * explicitly to be able to use DLT_PKTAP.
187 */
188 #if !XNU_TARGET_OS_OSX
189 static unsigned int bpf_wantpktap = 1;
190 #else /* XNU_TARGET_OS_OSX */
191 static unsigned int bpf_wantpktap = 0;
192 #endif /* XNU_TARGET_OS_OSX */
193 SYSCTL_UINT(_debug, OID_AUTO, bpf_wantpktap, CTLFLAG_RW | CTLFLAG_LOCKED,
194 &bpf_wantpktap, 0, "");
195
196 static int bpf_debug = 0;
197 SYSCTL_INT(_debug, OID_AUTO, bpf_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
198 &bpf_debug, 0, "");
199
200 static unsigned long bpf_trunc_overflow = 0;
201 SYSCTL_ULONG(_debug, OID_AUTO, bpf_trunc_overflow, CTLFLAG_RD | CTLFLAG_LOCKED,
202 &bpf_trunc_overflow, "");
203
204 static int bpf_hdr_comp_enable = 1;
205 SYSCTL_INT(_debug, OID_AUTO, bpf_hdr_comp_enable, CTLFLAG_RW | CTLFLAG_LOCKED,
206 &bpf_hdr_comp_enable, 1, "");
207
208 static int sysctl_bpf_stats SYSCTL_HANDLER_ARGS;
209 SYSCTL_PROC(_debug, OID_AUTO, bpf_stats, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
210 0, 0,
211 sysctl_bpf_stats, "S", "BPF statistics");
212
213 /*
214 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
215 * bpf_dtab holds pointer to the descriptors, indexed by minor device #
216 */
217 static struct bpf_if *bpf_iflist;
218 /*
219 * BSD now stores the bpf_d in the dev_t which is a struct
220 * on their system. Our dev_t is an int, so we still store
221 * the bpf_d in a separate table indexed by minor device #.
222 *
223 * The value stored in bpf_dtab[n] represent three states:
224 * NULL: device not opened
225 * BPF_DEV_RESERVED: device opening or closing
226 * other: device <n> opened with pointer to storage
227 */
228 static struct bpf_d *BPF_DEV_RESERVED = __unsafe_forge_single(struct bpf_d *, 1);
229 static unsigned int bpf_dtab_size = 0;
230 static struct bpf_d **__counted_by(bpf_dtab_size) bpf_dtab = NULL;
231 static unsigned int nbpfilter = 0;
232 static unsigned bpf_bpfd_cnt = 0;
233
234 static LCK_GRP_DECLARE(bpf_mlock_grp, "bpf");
235 static LCK_MTX_DECLARE(bpf_mlock_data, &bpf_mlock_grp);
236 static lck_mtx_t *const bpf_mlock = &bpf_mlock_data;
237
238 static int bpf_allocbufs(struct bpf_d *);
239 static errno_t bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
240 static int bpf_detachd(struct bpf_d *d, struct proc *);
241 static void bpf_freed(struct bpf_d *);
242 static int bpf_setif(struct bpf_d *, ifnet_t ifp, bool, bool, bool, struct proc *);
243 static void bpf_timed_out(void *, void *);
244 static void bpf_wakeup(struct bpf_d *);
245 static uint32_t get_pkt_trunc_len(struct bpf_packet *);
246 static void catchpacket(struct bpf_d *, struct bpf_packet *, u_int, int);
247 static void reset_d(struct bpf_d *);
248 static int bpf_setf(struct bpf_d *, u_int, user_addr_t, u_long);
249 static int bpf_getdltlist(struct bpf_d *, caddr_t __bidi_indexable, struct proc *);
250 static int bpf_setdlt(struct bpf_d *, u_int, struct proc *);
251 static int bpf_set_traffic_class(struct bpf_d *, int);
252 static void bpf_set_packet_service_class(struct mbuf *, int);
253
254 static void bpf_acquire_d(struct bpf_d *);
255 static void bpf_release_d(struct bpf_d *);
256
257 static int bpf_devsw_installed;
258
259 void bpf_init(void *unused);
260 static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m);
261
262 /*
263 * Darwin differs from BSD here, the following are static
264 * on BSD and not static on Darwin.
265 */
266 d_open_t bpfopen;
267 d_close_t bpfclose;
268 d_read_t bpfread;
269 d_write_t bpfwrite;
270 ioctl_fcn_t bpfioctl;
271 select_fcn_t bpfselect;
272
273 /* Darwin's cdevsw struct differs slightly from BSDs */
274 #define CDEV_MAJOR 23
275 static const struct cdevsw bpf_cdevsw = {
276 .d_open = bpfopen,
277 .d_close = bpfclose,
278 .d_read = bpfread,
279 .d_write = bpfwrite,
280 .d_ioctl = bpfioctl,
281 .d_stop = eno_stop,
282 .d_reset = eno_reset,
283 .d_ttys = NULL,
284 .d_select = bpfselect,
285 .d_mmap = eno_mmap,
286 .d_strategy = eno_strat,
287 .d_reserved_1 = eno_getc,
288 .d_reserved_2 = eno_putc,
289 .d_type = 0
290 };
291
292 #define SOCKADDR_HDR_LEN offsetof(struct sockaddr, sa_data)
293
294 static int
bpf_copy_uio_to_mbuf_packet(struct uio * auio,int bytes_to_copy,struct mbuf * top)295 bpf_copy_uio_to_mbuf_packet(struct uio *auio, int bytes_to_copy, struct mbuf *top)
296 {
297 int error = 0;
298
299 for (struct mbuf *m = top; m != NULL; m = m->m_next) {
300 int mlen;
301
302 if (m->m_flags & M_EXT) {
303 mlen = m->m_ext.ext_size - (int)M_LEADINGSPACE(m);
304 } else if (m->m_flags & M_PKTHDR) {
305 mlen = MHLEN - (int)M_LEADINGSPACE(m);
306 } else {
307 mlen = MLEN - (int)M_LEADINGSPACE(m);
308 }
309 int copy_len = imin((int)mlen, bytes_to_copy);
310
311 error = uiomove(mtod(m, caddr_t), (int)copy_len, auio);
312 if (error != 0) {
313 os_log(OS_LOG_DEFAULT, "bpf_copy_uio_to_mbuf_packet: len %d error %d",
314 copy_len, error);
315 goto done;
316 }
317 m->m_len = copy_len;
318 top->m_pkthdr.len += copy_len;
319
320 if (bytes_to_copy > copy_len) {
321 bytes_to_copy -= copy_len;
322 } else {
323 break;
324 }
325 }
326 done:
327 return error;
328 }
329
330 static inline void
bpf_set_bcast_mcast(mbuf_t m,struct ether_header * eh)331 bpf_set_bcast_mcast(mbuf_t m, struct ether_header * eh)
332 {
333 if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
334 if (_ether_cmp(etherbroadcastaddr, eh->ether_dhost) == 0) {
335 m->m_flags |= M_BCAST;
336 } else {
337 m->m_flags |= M_MCAST;
338 }
339 }
340 }
341
342 #if DEBUG | DEVELOPMENT
343 static void
bpf_log_bcast(const char * func,const char * ifname,uint16_t flags,bool hdrcmplt)344 bpf_log_bcast(const char * func, const char * ifname, uint16_t flags,
345 bool hdrcmplt)
346 {
347 const char * type;
348
349 if ((flags & M_BCAST) != 0) {
350 type = "broadcast";
351 } else if ((flags & M_MCAST) != 0) {
352 type = "multicast";
353 } else {
354 type = "unicast";
355 }
356 os_log(OS_LOG_DEFAULT, "%s %s %s hdrcmplt=%s", func, ifname, type,
357 hdrcmplt ? "true" : "false");
358 }
359 #endif /* DEBUG | DEVELOPMENT */
360
361 static int
bpf_movein(struct uio * uio,int copy_len,struct bpf_d * d,struct mbuf ** mp,struct sockaddr * sockp)362 bpf_movein(struct uio *uio, int copy_len, struct bpf_d *d, struct mbuf **mp,
363 struct sockaddr *sockp)
364 {
365 mbuf_ref_t m = NULL;
366 int error;
367 int len;
368 uint8_t sa_family;
369 int hlen = 0;
370 struct ifnet *ifp = d->bd_bif->bif_ifp;
371 int linktype = (int)d->bd_bif->bif_dlt;
372
373 switch (linktype) {
374 #if SLIP
375 case DLT_SLIP:
376 sa_family = AF_INET;
377 hlen = 0;
378 break;
379 #endif /* SLIP */
380
381 case DLT_EN10MB:
382 sa_family = AF_UNSPEC;
383 /* XXX Would MAXLINKHDR be better? */
384 hlen = sizeof(struct ether_header);
385 break;
386
387 #if FDDI
388 case DLT_FDDI:
389 #if defined(__FreeBSD__) || defined(__bsdi__)
390 sa_family = AF_IMPLINK;
391 hlen = 0;
392 #else
393 sa_family = AF_UNSPEC;
394 /* XXX 4(FORMAC)+6(dst)+6(src)+3(LLC)+5(SNAP) */
395 hlen = 24;
396 #endif
397 break;
398 #endif /* FDDI */
399
400 case DLT_RAW:
401 case DLT_NULL:
402 sa_family = AF_UNSPEC;
403 hlen = 0;
404 break;
405
406 #ifdef __FreeBSD__
407 case DLT_ATM_RFC1483:
408 /*
409 * en atm driver requires 4-byte atm pseudo header.
410 * though it isn't standard, vpi:vci needs to be
411 * specified anyway.
412 */
413 sa_family = AF_UNSPEC;
414 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
415 break;
416 #endif
417
418 case DLT_PPP:
419 sa_family = AF_UNSPEC;
420 hlen = 4; /* This should match PPP_HDRLEN */
421 break;
422
423 case DLT_APPLE_IP_OVER_IEEE1394:
424 sa_family = AF_UNSPEC;
425 hlen = sizeof(struct firewire_header);
426 break;
427
428 case DLT_IEEE802_11: /* IEEE 802.11 wireless */
429 sa_family = AF_IEEE80211;
430 hlen = 0;
431 break;
432
433 case DLT_IEEE802_11_RADIO:
434 sa_family = AF_IEEE80211;
435 hlen = 0;
436 break;
437
438 default:
439 return EIO;
440 }
441
442 if (sockp) {
443 /*
444 * Build a sockaddr based on the data link layer type.
445 * We do this at this level because the ethernet header
446 * is copied directly into the data field of the sockaddr.
447 * In the case of SLIP, there is no header and the packet
448 * is forwarded as is.
449 * Also, we are careful to leave room at the front of the mbuf
450 * for the link level header.
451 */
452 if ((hlen + SOCKADDR_HDR_LEN) > sockp->sa_len) {
453 return EIO;
454 }
455 sockp->sa_family = sa_family;
456 } else {
457 /*
458 * We're directly sending the packet data supplied by
459 * the user; we don't need to make room for the link
460 * header, and don't need the header length value any
461 * more, so set it to 0.
462 */
463 hlen = 0;
464 }
465
466 len = (int)uio_resid(uio);
467 if (len < copy_len) {
468 os_log(OS_LOG_DEFAULT, "bpfwrite: len %d if %s less than copy_len %d",
469 (unsigned)len, ifp->if_xname, copy_len);
470 return EMSGSIZE;
471 }
472 len = copy_len;
473 if (len < hlen || (unsigned)len > BPF_WRITE_MAX) {
474 os_log(OS_LOG_DEFAULT, "bpfwrite: bad len %d if %s",
475 (unsigned)len, ifp->if_xname);
476 return EMSGSIZE;
477 }
478 if (d->bd_write_size_max != 0) {
479 if ((len - hlen) > (d->bd_write_size_max + BPF_WRITE_LEEWAY)) {
480 os_log(OS_LOG_DEFAULT, "bpfwrite: len %u - hlen %u too big if %s write_size_max %u",
481 (unsigned)len, (unsigned)hlen, ifp->if_xname, d->bd_write_size_max);
482 }
483 } else if ((len - hlen) > (ifp->if_mtu + BPF_WRITE_LEEWAY)) {
484 os_log(OS_LOG_DEFAULT, "bpfwrite: len %u - hlen %u too big if %s mtu %u",
485 (unsigned)len, (unsigned)hlen, ifp->if_xname, ifp->if_mtu);
486 return EMSGSIZE;
487 }
488
489 /* drop lock while allocating mbuf and copying data */
490 lck_mtx_unlock(bpf_mlock);
491
492 error = mbuf_allocpacket(MBUF_WAITOK, len, NULL, &m);
493 if (error != 0) {
494 os_log(OS_LOG_DEFAULT,
495 "bpfwrite mbuf_allocpacket len %d error %d", len, error);
496 goto bad;
497 }
498 /*
499 * Make room for link header -- the packet length is 0 at this stage
500 */
501 if (hlen != 0) {
502 m->m_data += hlen; /* leading space */
503 error = uiomove((caddr_t)sockp->sa_data, hlen, uio);
504 if (error) {
505 os_log(OS_LOG_DEFAULT,
506 "bpfwrite uiomove hlen %d error %d", hlen, error);
507 goto bad;
508 }
509 len -= hlen;
510 if (linktype == DLT_EN10MB) {
511 struct ether_header * eh;
512
513 eh = (struct ether_header *)(void *)sockp->sa_data;
514 bpf_set_bcast_mcast(m, eh);
515 #if DEBUG || DEVELOPMENT
516 if (__improbable(bpf_debug != 0)) {
517 bpf_log_bcast(__func__, ifp->if_xname,
518 m->m_flags, false);
519 }
520 #endif /* DEBUG || DEVELOPMENT */
521 }
522 }
523 /*
524 * bpf_copy_uio_to_mbuf_packet() does set the length of each mbuf and adds it to
525 * the total packet length
526 */
527 error = bpf_copy_uio_to_mbuf_packet(uio, len, m);
528 if (error != 0) {
529 os_log(OS_LOG_DEFAULT,
530 "bpfwrite bpf_copy_uio_to_mbuf_packet error %d", error);
531 goto bad;
532 }
533
534 /* Check for multicast destination */
535 if (hlen == 0 && linktype == DLT_EN10MB) {
536 struct ether_header *eh;
537
538 eh = mtod(m, struct ether_header *);
539 bpf_set_bcast_mcast(m, eh);
540 #if DEBUG || DEVELOPMENT
541 if (__improbable(bpf_debug != 0)) {
542 bpf_log_bcast(__func__, ifp->if_xname,
543 m->m_flags, true);
544 }
545 #endif /* DEBUG || DEVELOPMENT */
546 }
547 *mp = m;
548
549 lck_mtx_lock(bpf_mlock);
550 return 0;
551 bad:
552 if (m != NULL) {
553 m_freem(m);
554 }
555 lck_mtx_lock(bpf_mlock);
556 return error;
557 }
558
559 static int
bpf_movein_batch(struct uio * uio,struct bpf_d * d,struct mbuf ** mp,struct sockaddr * sockp)560 bpf_movein_batch(struct uio *uio, struct bpf_d *d, struct mbuf **mp,
561 struct sockaddr *sockp)
562 {
563 int error = 0;
564 user_ssize_t resid;
565 int count = 0;
566 struct mbuf *last = NULL;
567
568 *mp = NULL;
569 while ((resid = uio_resid(uio)) >= sizeof(struct bpf_hdr)) {
570 struct bpf_hdr bpfhdr = {};
571 int bpf_hdr_min_len = offsetof(struct bpf_hdr, bh_hdrlen) + sizeof(bpfhdr.bh_hdrlen);
572 int padding_len;
573
574 error = uiomove((caddr_t)&bpfhdr, bpf_hdr_min_len, uio);
575 if (error != 0) {
576 os_log(OS_LOG_DEFAULT, "bpf_movein_batch uiomove error %d", error);
577 break;
578 }
579 /*
580 * Buffer validation:
581 * - ignore bh_tstamp
582 * - bh_hdrlen must fit
583 * - bh_caplen and bh_datalen must be equal
584 */
585 if (bpfhdr.bh_hdrlen < bpf_hdr_min_len) {
586 error = EINVAL;
587 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_hdrlen %u too small",
588 bpfhdr.bh_hdrlen);
589 break;
590 }
591 if (bpfhdr.bh_caplen != bpfhdr.bh_datalen) {
592 error = EINVAL;
593 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_caplen %u != bh_datalen %u",
594 bpfhdr.bh_caplen, bpfhdr.bh_datalen);
595 break;
596 }
597 if (bpfhdr.bh_hdrlen > resid) {
598 error = EINVAL;
599 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_hdrlen %u too large",
600 bpfhdr.bh_hdrlen);
601 break;
602 }
603
604 /*
605 * Ignore additional bytes in the header
606 */
607 padding_len = bpfhdr.bh_hdrlen - bpf_hdr_min_len;
608 if (padding_len > 0) {
609 uio_update(uio, padding_len);
610 }
611
612 /* skip empty packets */
613 if (bpfhdr.bh_caplen > 0) {
614 mbuf_ref_t m;
615
616 /*
617 * For time being assume all packets have same destination
618 */
619 error = bpf_movein(uio, bpfhdr.bh_caplen, d, &m, sockp);
620 if (error != 0) {
621 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bpf_movein error %d",
622 error);
623 break;
624 }
625 count += 1;
626
627 if (last == NULL) {
628 *mp = m;
629 } else {
630 last->m_nextpkt = m;
631 }
632 last = m;
633 }
634
635 /*
636 * Each BPF packet is padded for alignment
637 */
638 padding_len = BPF_WORDALIGN(bpfhdr.bh_hdrlen + bpfhdr.bh_caplen) - (bpfhdr.bh_hdrlen + bpfhdr.bh_caplen);
639 if (padding_len > 0) {
640 uio_update(uio, padding_len);
641 }
642 }
643
644 if (error != 0) {
645 if (*mp != NULL) {
646 m_freem_list(*mp);
647 *mp = NULL;
648 }
649 }
650 return error;
651 }
652
653 /*
654 * The dynamic addition of a new device node must block all processes that
655 * are opening the last device so that no process will get an unexpected
656 * ENOENT
657 */
658 static void
bpf_make_dev_t(int maj)659 bpf_make_dev_t(int maj)
660 {
661 static int bpf_growing = 0;
662 unsigned int cur_size = nbpfilter, i;
663
664 if (nbpfilter >= BPF_MAX_DEVICES) {
665 return;
666 }
667
668 while (bpf_growing) {
669 /* Wait until new device has been created */
670 (void) tsleep((caddr_t)&bpf_growing, PZERO, "bpf_growing", 0);
671 }
672 if (nbpfilter > cur_size) {
673 /* other thread grew it already */
674 return;
675 }
676 bpf_growing = 1;
677
678 /* need to grow bpf_dtab first */
679 if (nbpfilter == bpf_dtab_size) {
680 unsigned int new_dtab_size;
681 struct bpf_d **new_dtab = NULL;
682
683 new_dtab_size = bpf_dtab_size + NBPFILTER;
684 new_dtab = krealloc_type(struct bpf_d *,
685 bpf_dtab_size, new_dtab_size, bpf_dtab, Z_WAITOK | Z_ZERO);
686
687 if (new_dtab == 0) {
688 os_log_error(OS_LOG_DEFAULT, "bpf_make_dev_t: malloc bpf_dtab failed");
689 goto done;
690 }
691 bpf_dtab = new_dtab;
692 bpf_dtab_size = new_dtab_size;
693 }
694 i = nbpfilter++;
695 (void) devfs_make_node(makedev(maj, i),
696 DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0600,
697 "bpf%d", i);
698 done:
699 bpf_growing = 0;
700 wakeup((caddr_t)&bpf_growing);
701 }
702
703 /*
704 * Attach file to the bpf interface, i.e. make d listen on bp.
705 */
706 static errno_t
bpf_attachd(struct bpf_d * d,struct bpf_if * bp)707 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
708 {
709 int first = bp->bif_dlist == NULL;
710 int error = 0;
711
712 /*
713 * Point d at bp, and add d to the interface's list of listeners.
714 * Finally, point the driver's bpf cookie at the interface so
715 * it will divert packets to bpf.
716 */
717 d->bd_bif = bp;
718 d->bd_next = bp->bif_dlist;
719 bp->bif_dlist = d;
720 bpf_bpfd_cnt++;
721
722 /*
723 * Take a reference on the device even if an error is returned
724 * because we keep the device in the interface's list of listeners
725 */
726 bpf_acquire_d(d);
727
728 if (first) {
729 struct ifnet *ifp = NULL;
730 bool is_primary_dlt = false;
731 bpf_tap_func tap_func = NULL;
732
733 /* Find the default bpf entry for this ifp */
734 if (bp->bif_ifp->if_bpf == NULL) {
735 struct bpf_if *tmp, *primary = NULL;
736
737 for (tmp = bpf_iflist; tmp; tmp = tmp->bif_next) {
738 if (tmp->bif_ifp == bp->bif_ifp) {
739 primary = tmp;
740 break;
741 }
742 }
743 bp->bif_ifp->if_bpf = primary;
744 }
745
746 ifp = bp->bif_ifp;
747
748 /* Only call dlil_set_bpf_tap for primary dlt */
749 if (bp->bif_ifp->if_bpf == bp) {
750 is_primary_dlt = true;
751 }
752
753 tap_func = bp->bif_tap;
754
755 /* Unlock to workaround lock ordering issue with driver */
756 lck_mtx_unlock(bpf_mlock);
757
758 if (is_primary_dlt) {
759 dlil_set_bpf_tap(ifp, BPF_TAP_INPUT_OUTPUT,
760 bpf_tap_callback);
761 }
762
763 if (tap_func != NULL) {
764 error = tap_func(bp->bif_ifp, bp->bif_dlt,
765 BPF_TAP_INPUT_OUTPUT);
766 }
767 lck_mtx_lock(bpf_mlock);
768 }
769
770 /*
771 * Reset the detach flags in case we previously detached an interface
772 */
773 d->bd_flags &= ~(BPF_DETACHING | BPF_DETACHED);
774
775 if (bp->bif_dlt == DLT_PKTAP) {
776 d->bd_flags |= BPF_FINALIZE_PKTAP;
777 } else {
778 d->bd_flags &= ~BPF_FINALIZE_PKTAP;
779 }
780 return error;
781 }
782
783 /*
784 * Detach a file from its interface.
785 *
786 * Return 1 if was closed by some thread, 0 otherwise
787 */
788 static int
bpf_detachd(struct bpf_d * d,struct proc * proc)789 bpf_detachd(struct bpf_d *d, struct proc *proc)
790 {
791 struct bpf_d **p;
792 struct bpf_if *bp;
793 struct ifnet *ifp;
794 uint32_t dlt;
795 bpf_tap_func disable_tap = NULL;
796 uint8_t bd_promisc;
797 bool is_primary_dlt = false;
798
799 int bpf_closed = d->bd_flags & BPF_CLOSING;
800 /*
801 * Some other thread already detached
802 */
803 if ((d->bd_flags & (BPF_DETACHED | BPF_DETACHING)) != 0) {
804 goto done;
805 }
806 /*
807 * This thread is doing the detach
808 */
809 d->bd_flags |= BPF_DETACHING;
810
811 ifp = d->bd_bif->bif_ifp;
812 bp = d->bd_bif;
813
814 /* Remove d from the interface's descriptor list. */
815 p = &bp->bif_dlist;
816 while (*p != d) {
817 p = &(*p)->bd_next;
818 if (*p == 0) {
819 panic("bpf_detachd: descriptor not in list");
820 }
821 }
822 *p = (*p)->bd_next;
823 bpf_bpfd_cnt--;
824 disable_tap = NULL;
825 if (bp->bif_dlist == 0) {
826 /*
827 * Let the driver know that there are no more listeners.
828 */
829 /* Only call dlil_set_bpf_tap for primary dlt */
830 if (bp->bif_ifp->if_bpf == bp) {
831 is_primary_dlt = true;
832 }
833
834 disable_tap = bp->bif_tap;
835 if (disable_tap) {
836 dlt = bp->bif_dlt;
837 }
838
839 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
840 if (bp->bif_ifp == ifp && bp->bif_dlist != 0) {
841 break;
842 }
843 }
844 if (bp == NULL) {
845 ifp->if_bpf = NULL;
846 }
847 }
848 d->bd_bif = NULL;
849
850 /*
851 * Stop disabling input
852 */
853 if ((d->bd_flags & BPF_DIVERT_IN) != 0) {
854 if_clear_xflags(ifp, IFXF_DISABLE_INPUT);
855 d->bd_flags &= ~BPF_DIVERT_IN;
856
857 os_log(OS_LOG_DEFAULT,
858 "bpf_detachd: bpf%d %s disable input 0",
859 d->bd_dev_minor, if_name(ifp));
860 }
861
862 /*
863 * Check if this descriptor had requested promiscuous mode.
864 * If so, turn it off.
865 */
866 bd_promisc = d->bd_promisc;
867 d->bd_promisc = 0;
868
869 lck_mtx_unlock(bpf_mlock);
870
871 if (is_primary_dlt) {
872 dlil_set_bpf_tap(ifp, BPF_TAP_DISABLE, NULL);
873 }
874
875 if (disable_tap) {
876 disable_tap(ifp, dlt, BPF_TAP_DISABLE);
877 }
878
879 if (bd_promisc) {
880 if (ifnet_set_promiscuous(ifp, 0)) {
881 /*
882 * Something is really wrong if we were able to put
883 * the driver into promiscuous mode, but can't
884 * take it out.
885 * Most likely the network interface is gone.
886 */
887 os_log_error(OS_LOG_DEFAULT,
888 "%s: bpf%d ifnet_set_promiscuous %s failed",
889 __func__, d->bd_dev_minor, if_name(ifp));
890 }
891 }
892
893 lck_mtx_lock(bpf_mlock);
894
895 /*
896 * Wake up other thread that are waiting for this thread to finish
897 * detaching
898 */
899 d->bd_flags &= ~BPF_DETACHING;
900 d->bd_flags |= BPF_DETACHED;
901
902 /* Refresh the local variable as d could have been modified */
903 bpf_closed = d->bd_flags & BPF_CLOSING;
904
905 os_log(OS_LOG_DEFAULT, "bpf%d%s detached from %s fcount %llu dcount %llu by %s:%u",
906 d->bd_dev_minor, bpf_closed ? " closed and" : "", if_name(ifp),
907 d->bd_fcount, d->bd_dcount, proc_name_address(proc), proc_pid(proc));
908
909 /*
910 * Note that We've kept the reference because we may have dropped
911 * the lock when turning off promiscuous mode
912 */
913 bpf_release_d(d);
914 done:
915 /*
916 * Let the caller know the bpf_d is closed
917 */
918 if (bpf_closed) {
919 return 1;
920 } else {
921 return 0;
922 }
923 }
924
925 /*
926 * Start asynchronous timer, if necessary.
927 * Must be called with bpf_mlock held.
928 */
929 static void
bpf_start_timer(struct bpf_d * d)930 bpf_start_timer(struct bpf_d *d)
931 {
932 uint64_t deadline;
933 struct timeval tv;
934
935 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
936 tv.tv_sec = d->bd_rtout / hz;
937 tv.tv_usec = (d->bd_rtout % hz) * tick;
938
939 clock_interval_to_deadline(
940 (uint32_t)tv.tv_sec * USEC_PER_SEC + tv.tv_usec,
941 NSEC_PER_USEC, &deadline);
942 /*
943 * The state is BPF_IDLE, so the timer hasn't
944 * been started yet, and hasn't gone off yet;
945 * there is no thread call scheduled, so this
946 * won't change the schedule.
947 *
948 * XXX - what if, by the time it gets entered,
949 * the deadline has already passed?
950 */
951 thread_call_enter_delayed(d->bd_thread_call, deadline);
952 d->bd_state = BPF_WAITING;
953 }
954 }
955
956 /*
957 * Cancel asynchronous timer.
958 * Must be called with bpf_mlock held.
959 */
960 static boolean_t
bpf_stop_timer(struct bpf_d * d)961 bpf_stop_timer(struct bpf_d *d)
962 {
963 /*
964 * If the timer has already gone off, this does nothing.
965 * Our caller is expected to set d->bd_state to BPF_IDLE,
966 * with the bpf_mlock, after we are called. bpf_timed_out()
967 * also grabs bpf_mlock, so, if the timer has gone off and
968 * bpf_timed_out() hasn't finished, it's waiting for the
969 * lock; when this thread releases the lock, it will
970 * find the state is BPF_IDLE, and just release the
971 * lock and return.
972 */
973 return thread_call_cancel(d->bd_thread_call);
974 }
975
976 void
bpf_acquire_d(struct bpf_d * d)977 bpf_acquire_d(struct bpf_d *d)
978 {
979 void *__single lr_saved = __unsafe_forge_single(void *, __builtin_return_address(0));
980
981 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
982
983 d->bd_refcnt += 1;
984
985 d->bd_ref_lr[d->bd_next_ref_lr] = lr_saved;
986 d->bd_next_ref_lr = (d->bd_next_ref_lr + 1) % BPF_REF_HIST;
987 }
988
989 void
bpf_release_d(struct bpf_d * d)990 bpf_release_d(struct bpf_d *d)
991 {
992 void *__single lr_saved = __unsafe_forge_single(void *, __builtin_return_address(0));
993
994 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
995
996 if (d->bd_refcnt <= 0) {
997 panic("%s: %p refcnt <= 0", __func__, d);
998 }
999
1000 d->bd_refcnt -= 1;
1001
1002 d->bd_unref_lr[d->bd_next_unref_lr] = lr_saved;
1003 d->bd_next_unref_lr = (d->bd_next_unref_lr + 1) % BPF_REF_HIST;
1004
1005 if (d->bd_refcnt == 0) {
1006 /* Assert the device is detached */
1007 if ((d->bd_flags & BPF_DETACHED) == 0) {
1008 panic("%s: %p BPF_DETACHED not set", __func__, d);
1009 }
1010
1011 kfree_type(struct bpf_d, d);
1012 }
1013 }
1014
1015 /*
1016 * Open ethernet device. Returns ENXIO for illegal minor device number,
1017 * EBUSY if file is open by another process.
1018 */
1019 /* ARGSUSED */
1020 int
bpfopen(dev_t dev,int flags,__unused int fmt,struct proc * proc)1021 bpfopen(dev_t dev, int flags, __unused int fmt,
1022 struct proc *proc)
1023 {
1024 struct bpf_d *d;
1025
1026 lck_mtx_lock(bpf_mlock);
1027 if ((unsigned int) minor(dev) >= nbpfilter) {
1028 lck_mtx_unlock(bpf_mlock);
1029 return ENXIO;
1030 }
1031 /*
1032 * New device nodes are created on demand when opening the last one.
1033 * The programming model is for processes to loop on the minor starting
1034 * at 0 as long as EBUSY is returned. The loop stops when either the
1035 * open succeeds or an error other that EBUSY is returned. That means
1036 * that bpf_make_dev_t() must block all processes that are opening the
1037 * last node. If not all processes are blocked, they could unexpectedly
1038 * get ENOENT and abort their opening loop.
1039 */
1040 if ((unsigned int) minor(dev) == (nbpfilter - 1)) {
1041 bpf_make_dev_t(major(dev));
1042 }
1043
1044 /*
1045 * Each minor can be opened by only one process. If the requested
1046 * minor is in use, return EBUSY.
1047 *
1048 * Important: bpfopen() and bpfclose() have to check and set the status
1049 * of a device in the same lockin context otherwise the device may be
1050 * leaked because the vnode use count will be unpextectly greater than 1
1051 * when close() is called.
1052 */
1053 if (bpf_dtab[minor(dev)] == NULL) {
1054 /* Reserve while opening */
1055 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED;
1056 } else {
1057 lck_mtx_unlock(bpf_mlock);
1058 return EBUSY;
1059 }
1060 d = kalloc_type(struct bpf_d, Z_WAITOK | Z_ZERO);
1061 if (d == NULL) {
1062 /* this really is a catastrophic failure */
1063 os_log_error(OS_LOG_DEFAULT,
1064 "bpfopen: bpf%d kalloc_type bpf_d failed", minor(dev));
1065 bpf_dtab[minor(dev)] = NULL;
1066 lck_mtx_unlock(bpf_mlock);
1067 return ENOMEM;
1068 }
1069
1070 /* Mark "in use" and do most initialization. */
1071 bpf_acquire_d(d);
1072 d->bd_bufsize = bpf_bufsize;
1073 d->bd_sig = SIGIO;
1074 d->bd_direction = BPF_D_INOUT;
1075 d->bd_oflags = flags;
1076 d->bd_state = BPF_IDLE;
1077 d->bd_traffic_class = SO_TC_BE;
1078 d->bd_flags |= BPF_DETACHED;
1079 if (bpf_wantpktap) {
1080 d->bd_flags |= BPF_WANT_PKTAP;
1081 } else {
1082 d->bd_flags &= ~BPF_WANT_PKTAP;
1083 }
1084
1085 d->bd_thread_call = thread_call_allocate(bpf_timed_out, d);
1086 if (d->bd_thread_call == NULL) {
1087 os_log_error(OS_LOG_DEFAULT, "bpfopen: bpf%d malloc thread call failed",
1088 minor(dev));
1089 bpf_dtab[minor(dev)] = NULL;
1090 bpf_release_d(d);
1091 lck_mtx_unlock(bpf_mlock);
1092
1093 return ENOMEM;
1094 }
1095
1096 /* Use the proc pointer for comparaison so no need to take a reference */
1097 d->bd_opened_by = proc;
1098 uuid_generate(d->bd_uuid);
1099 d->bd_pid = proc_pid(proc);
1100
1101 d->bd_dev_minor = minor(dev);
1102 bpf_dtab[minor(dev)] = d; /* Mark opened */
1103 lck_mtx_unlock(bpf_mlock);
1104
1105 if (bpf_debug) {
1106 os_log(OS_LOG_DEFAULT, "bpf%u opened by %s.%u",
1107 d->bd_dev_minor, proc_name_address(proc), d->bd_pid);
1108 }
1109 return 0;
1110 }
1111
1112 /*
1113 * Close the descriptor by detaching it from its interface,
1114 * deallocating its buffers, and marking it free.
1115 */
1116 /* ARGSUSED */
1117 int
bpfclose(dev_t dev,__unused int flags,__unused int fmt,struct proc * proc)1118 bpfclose(dev_t dev, __unused int flags, __unused int fmt,
1119 struct proc *proc)
1120 {
1121 struct bpf_d *d;
1122
1123 /* Take BPF lock to ensure no other thread is using the device */
1124 lck_mtx_lock(bpf_mlock);
1125
1126 d = bpf_dtab[minor(dev)];
1127 if (d == NULL || d == BPF_DEV_RESERVED) {
1128 lck_mtx_unlock(bpf_mlock);
1129 return ENXIO;
1130 }
1131
1132 /*
1133 * Other threads may call bpd_detachd() if we drop the bpf_mlock
1134 */
1135 d->bd_flags |= BPF_CLOSING;
1136
1137 if (bpf_debug != 0) {
1138 os_log(OS_LOG_DEFAULT, "%s: bpf%d",
1139 __func__, d->bd_dev_minor);
1140 }
1141
1142 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED; /* Reserve while closing */
1143
1144 /*
1145 * Deal with any in-progress timeouts.
1146 */
1147 switch (d->bd_state) {
1148 case BPF_IDLE:
1149 /*
1150 * Not waiting for a timeout, and no timeout happened.
1151 */
1152 break;
1153
1154 case BPF_WAITING:
1155 /*
1156 * Waiting for a timeout.
1157 * Cancel any timer that has yet to go off,
1158 * and mark the state as "closing".
1159 * Then drop the lock to allow any timers that
1160 * *have* gone off to run to completion, and wait
1161 * for them to finish.
1162 */
1163 if (!bpf_stop_timer(d)) {
1164 /*
1165 * There was no pending call, so the call must
1166 * have been in progress. Wait for the call to
1167 * complete; we have to drop the lock while
1168 * waiting. to let the in-progrss call complete
1169 */
1170 d->bd_state = BPF_DRAINING;
1171 while (d->bd_state == BPF_DRAINING) {
1172 msleep((caddr_t)d, bpf_mlock, PRINET,
1173 "bpfdraining", NULL);
1174 }
1175 }
1176 d->bd_state = BPF_IDLE;
1177 break;
1178
1179 case BPF_TIMED_OUT:
1180 /*
1181 * Timer went off, and the timeout routine finished.
1182 */
1183 d->bd_state = BPF_IDLE;
1184 break;
1185
1186 case BPF_DRAINING:
1187 /*
1188 * Another thread is blocked on a close waiting for
1189 * a timeout to finish.
1190 * This "shouldn't happen", as the first thread to enter
1191 * bpfclose() will set bpf_dtab[minor(dev)] to 1, and
1192 * all subsequent threads should see that and fail with
1193 * ENXIO.
1194 */
1195 panic("Two threads blocked in a BPF close");
1196 break;
1197 }
1198
1199 if (d->bd_bif) {
1200 bpf_detachd(d, proc);
1201 }
1202 selthreadclear(&d->bd_sel);
1203 thread_call_free(d->bd_thread_call);
1204
1205 while (d->bd_hbuf_read || d->bd_hbuf_write) {
1206 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfclose", NULL);
1207 }
1208
1209 if (bpf_debug) {
1210 os_log(OS_LOG_DEFAULT,
1211 "bpf%u closed by %s.%u dcount %llu fcount %llu ccount %llu",
1212 d->bd_dev_minor, proc_name_address(proc), proc_pid(proc),
1213 d->bd_dcount, d->bd_fcount, d->bd_bcs.bcs_count_compressed_prefix);
1214 }
1215
1216 bpf_freed(d);
1217
1218 /* Mark free in same context as bpfopen comes to check */
1219 bpf_dtab[minor(dev)] = NULL; /* Mark closed */
1220
1221 bpf_release_d(d);
1222
1223 lck_mtx_unlock(bpf_mlock);
1224
1225 return 0;
1226 }
1227
1228 #define BPF_SLEEP bpf_sleep
1229
1230 static int
bpf_sleep(struct bpf_d * d,int pri,const char * wmesg,int timo)1231 bpf_sleep(struct bpf_d *d, int pri, const char *wmesg, int timo)
1232 {
1233 u_int64_t abstime = 0;
1234
1235 if (timo != 0) {
1236 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
1237 }
1238
1239 return msleep1((caddr_t)d, bpf_mlock, pri, wmesg, abstime);
1240 }
1241
1242 static void
bpf_finalize_pktap(struct bpf_hdr * hp,struct pktap_header * pktaphdr)1243 bpf_finalize_pktap(struct bpf_hdr *hp, struct pktap_header *pktaphdr)
1244 {
1245 if (pktaphdr->pth_flags & PTH_FLAG_V2_HDR) {
1246 struct pktap_v2_hdr *pktap_v2_hdr;
1247
1248 pktap_v2_hdr = (struct pktap_v2_hdr *)pktaphdr;
1249
1250 if (pktap_v2_hdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1251 pktap_v2_finalize_proc_info(pktap_v2_hdr);
1252 }
1253 } else {
1254 if (pktaphdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1255 pktap_finalize_proc_info(pktaphdr);
1256 }
1257
1258 if (pktaphdr->pth_flags & PTH_FLAG_TSTAMP) {
1259 hp->bh_tstamp.tv_sec = pktaphdr->pth_tstamp.tv_sec;
1260 hp->bh_tstamp.tv_usec = pktaphdr->pth_tstamp.tv_usec;
1261 }
1262 }
1263 }
1264
1265 /*
1266 * Rotate the packet buffers in descriptor d. Move the store buffer
1267 * into the hold slot, and the free buffer into the store slot.
1268 * Zero the length of the new store buffer.
1269 *
1270 * Note: in head drop mode, the hold buffer can be dropped so the fist packet of the
1271 * store buffer cannot be compressed as it otherwise would refer to deleted data
1272 * in a dropped hold buffer that the reader process does know about
1273 */
1274 #define ROTATE_BUFFERS(d) do { \
1275 if (d->bd_hbuf_read) \
1276 panic("rotating bpf buffers during read"); \
1277 (d)->bd_hbuf = (d)->bd_sbuf; \
1278 (d)->bd_hlen = (d)->bd_slen; \
1279 (d)->bd_hcnt = (d)->bd_scnt; \
1280 (d)->bd_sbuf = (d)->bd_fbuf; \
1281 (d)->bd_slen = 0; \
1282 (d)->bd_scnt = 0; \
1283 (d)->bd_fbuf = NULL; \
1284 if ((d)->bd_headdrop != 0) \
1285 (d)->bd_prev_slen = 0; \
1286 } while(false)
1287
1288 /*
1289 * bpfread - read next chunk of packets from buffers
1290 */
1291 int
bpfread(dev_t dev,struct uio * uio,int ioflag)1292 bpfread(dev_t dev, struct uio *uio, int ioflag)
1293 {
1294 struct bpf_d *d;
1295 caddr_t hbuf;
1296 int timed_out, hbuf_len;
1297 int error;
1298 int flags;
1299
1300 lck_mtx_lock(bpf_mlock);
1301
1302 d = bpf_dtab[minor(dev)];
1303 if (d == NULL || d == BPF_DEV_RESERVED ||
1304 (d->bd_flags & BPF_CLOSING) != 0) {
1305 lck_mtx_unlock(bpf_mlock);
1306 return ENXIO;
1307 }
1308
1309 bpf_acquire_d(d);
1310
1311 /*
1312 * Restrict application to use a buffer the same size as
1313 * as kernel buffers.
1314 */
1315 if (uio_resid(uio) != d->bd_bufsize) {
1316 bpf_release_d(d);
1317 lck_mtx_unlock(bpf_mlock);
1318 return EINVAL;
1319 }
1320
1321 if (d->bd_state == BPF_WAITING) {
1322 bpf_stop_timer(d);
1323 }
1324
1325 timed_out = (d->bd_state == BPF_TIMED_OUT);
1326 d->bd_state = BPF_IDLE;
1327
1328 while (d->bd_hbuf_read) {
1329 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfread", NULL);
1330 }
1331
1332 if ((d->bd_flags & BPF_CLOSING) != 0) {
1333 bpf_release_d(d);
1334 lck_mtx_unlock(bpf_mlock);
1335 return ENXIO;
1336 }
1337 /*
1338 * If the hold buffer is empty, then do a timed sleep, which
1339 * ends when the timeout expires or when enough packets
1340 * have arrived to fill the store buffer.
1341 */
1342 while (d->bd_hbuf == 0) {
1343 if ((d->bd_immediate || timed_out || (ioflag & IO_NDELAY)) &&
1344 d->bd_slen != 0) {
1345 /*
1346 * We're in immediate mode, or are reading
1347 * in non-blocking mode, or a timer was
1348 * started before the read (e.g., by select()
1349 * or poll()) and has expired and a packet(s)
1350 * either arrived since the previous
1351 * read or arrived while we were asleep.
1352 * Rotate the buffers and return what's here.
1353 */
1354 ROTATE_BUFFERS(d);
1355 break;
1356 }
1357
1358 /*
1359 * No data is available, check to see if the bpf device
1360 * is still pointed at a real interface. If not, return
1361 * ENXIO so that the userland process knows to rebind
1362 * it before using it again.
1363 */
1364 if (d->bd_bif == NULL) {
1365 bpf_release_d(d);
1366 lck_mtx_unlock(bpf_mlock);
1367 return ENXIO;
1368 }
1369 if (ioflag & IO_NDELAY) {
1370 bpf_release_d(d);
1371 lck_mtx_unlock(bpf_mlock);
1372 return EWOULDBLOCK;
1373 }
1374 error = BPF_SLEEP(d, PRINET | PCATCH, "bpf", d->bd_rtout);
1375 /*
1376 * Make sure device is still opened
1377 */
1378 if ((d->bd_flags & BPF_CLOSING) != 0) {
1379 bpf_release_d(d);
1380 lck_mtx_unlock(bpf_mlock);
1381 return ENXIO;
1382 }
1383
1384 while (d->bd_hbuf_read) {
1385 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_read",
1386 NULL);
1387 }
1388
1389 if ((d->bd_flags & BPF_CLOSING) != 0) {
1390 bpf_release_d(d);
1391 lck_mtx_unlock(bpf_mlock);
1392 return ENXIO;
1393 }
1394
1395 if (error == EINTR || error == ERESTART) {
1396 if (d->bd_hbuf != NULL) {
1397 /*
1398 * Because we msleep, the hold buffer might
1399 * be filled when we wake up. Avoid rotating
1400 * in this case.
1401 */
1402 break;
1403 }
1404 if (d->bd_slen != 0) {
1405 /*
1406 * Sometimes we may be interrupted often and
1407 * the sleep above will not timeout.
1408 * Regardless, we should rotate the buffers
1409 * if there's any new data pending and
1410 * return it.
1411 */
1412 ROTATE_BUFFERS(d);
1413 break;
1414 }
1415 bpf_release_d(d);
1416 lck_mtx_unlock(bpf_mlock);
1417 if (error == ERESTART) {
1418 os_log(OS_LOG_DEFAULT, "%s: bpf%d ERESTART to EINTR",
1419 __func__, d->bd_dev_minor);
1420 error = EINTR;
1421 }
1422 return error;
1423 }
1424 if (error == EWOULDBLOCK) {
1425 /*
1426 * On a timeout, return what's in the buffer,
1427 * which may be nothing. If there is something
1428 * in the store buffer, we can rotate the buffers.
1429 */
1430 if (d->bd_hbuf) {
1431 /*
1432 * We filled up the buffer in between
1433 * getting the timeout and arriving
1434 * here, so we don't need to rotate.
1435 */
1436 break;
1437 }
1438
1439 if (d->bd_slen == 0) {
1440 bpf_release_d(d);
1441 lck_mtx_unlock(bpf_mlock);
1442 return 0;
1443 }
1444 ROTATE_BUFFERS(d);
1445 break;
1446 }
1447 }
1448 /*
1449 * At this point, we know we have something in the hold slot.
1450 */
1451
1452 /*
1453 * Set the hold buffer read. So we do not
1454 * rotate the buffers until the hold buffer
1455 * read is complete. Also to avoid issues resulting
1456 * from page faults during disk sleep (<rdar://problem/13436396>).
1457 */
1458 d->bd_hbuf_read = true;
1459 hbuf = d->bd_hbuf;
1460 hbuf_len = d->bd_hlen;
1461 flags = d->bd_flags;
1462 d->bd_bcs.bcs_total_read += d->bd_hcnt;
1463 lck_mtx_unlock(bpf_mlock);
1464
1465 /*
1466 * Before we move data to userland, we fill out the extended
1467 * header fields.
1468 */
1469 if (flags & BPF_EXTENDED_HDR) {
1470 char *p;
1471
1472 p = hbuf;
1473 while (p < hbuf + hbuf_len) {
1474 struct bpf_hdr_ext *ehp;
1475 uint32_t flowid;
1476 struct so_procinfo soprocinfo;
1477 int found = 0;
1478
1479 ehp = (struct bpf_hdr_ext *)(void *)p;
1480 if ((flowid = ehp->bh_flowid) != 0) {
1481 if (ehp->bh_flags & BPF_HDR_EXT_FLAGS_TCP) {
1482 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_TCP;
1483 found = inp_findinpcb_procinfo(&tcbinfo,
1484 flowid, &soprocinfo);
1485 } else if (ehp->bh_flags == BPF_HDR_EXT_FLAGS_UDP) {
1486 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_UDP;
1487 found = inp_findinpcb_procinfo(&udbinfo,
1488 flowid, &soprocinfo);
1489 }
1490 if (found == 1) {
1491 ehp->bh_pid = soprocinfo.spi_pid;
1492 strbufcpy(ehp->bh_comm,
1493 soprocinfo.spi_proc_name);
1494 }
1495 ehp->bh_flowid = 0;
1496 }
1497
1498 if ((flags & BPF_FINALIZE_PKTAP) != 0 && ehp->bh_complen == 0) {
1499 struct pktap_header *pktaphdr;
1500
1501 pktaphdr = (struct pktap_header *)(void *)
1502 (p + BPF_WORDALIGN(ehp->bh_hdrlen));
1503
1504 bpf_finalize_pktap((struct bpf_hdr *) ehp,
1505 pktaphdr);
1506 }
1507 p += BPF_WORDALIGN(ehp->bh_hdrlen + ehp->bh_caplen);
1508 }
1509 } else if (flags & BPF_FINALIZE_PKTAP) {
1510 char *p;
1511
1512 p = hbuf;
1513
1514 while (p < hbuf + hbuf_len) {
1515 struct bpf_hdr *hp;
1516 struct pktap_header *pktaphdr;
1517
1518 hp = (struct bpf_hdr *)(void *)p;
1519
1520 /*
1521 * Cannot finalize a compressed pktap header as we may not have
1522 * all the fields present
1523 */
1524 if (d->bd_flags & BPF_COMP_ENABLED) {
1525 struct bpf_comp_hdr *hcp;
1526
1527 hcp = (struct bpf_comp_hdr *)(void *)p;
1528
1529 if (hcp->bh_complen != 0) {
1530 p += BPF_WORDALIGN(hcp->bh_hdrlen + hcp->bh_caplen);
1531 continue;
1532 }
1533 }
1534
1535 pktaphdr = (struct pktap_header *)(void *)
1536 (p + BPF_WORDALIGN(hp->bh_hdrlen));
1537
1538 bpf_finalize_pktap(hp, pktaphdr);
1539
1540 p += BPF_WORDALIGN(hp->bh_hdrlen + hp->bh_caplen);
1541 }
1542 }
1543
1544 /*
1545 * Move data from hold buffer into user space.
1546 * We know the entire buffer is transferred since
1547 * we checked above that the read buffer is bpf_bufsize bytes.
1548 */
1549 error = uiomove(hbuf, hbuf_len, uio);
1550
1551 lck_mtx_lock(bpf_mlock);
1552 /*
1553 * Make sure device is still opened
1554 */
1555 if ((d->bd_flags & BPF_CLOSING) != 0) {
1556 bpf_release_d(d);
1557 lck_mtx_unlock(bpf_mlock);
1558 return ENXIO;
1559 }
1560
1561 d->bd_hbuf_read = false;
1562 d->bd_fbuf = d->bd_hbuf;
1563 d->bd_hbuf = NULL;
1564 d->bd_hlen = 0;
1565 d->bd_hcnt = 0;
1566 wakeup((caddr_t)d);
1567
1568 bpf_release_d(d);
1569 lck_mtx_unlock(bpf_mlock);
1570 return error;
1571 }
1572
1573 /*
1574 * If there are processes sleeping on this descriptor, wake them up.
1575 */
1576 static void
bpf_wakeup(struct bpf_d * d)1577 bpf_wakeup(struct bpf_d *d)
1578 {
1579 if (d->bd_state == BPF_WAITING) {
1580 bpf_stop_timer(d);
1581 d->bd_state = BPF_IDLE;
1582 }
1583 wakeup((caddr_t)d);
1584 if (d->bd_async && d->bd_sig && d->bd_sigio) {
1585 pgsigio(d->bd_sigio, d->bd_sig);
1586 }
1587
1588 selwakeup(&d->bd_sel);
1589 if ((d->bd_flags & BPF_KNOTE)) {
1590 KNOTE(&d->bd_sel.si_note, 1);
1591 }
1592 }
1593
1594 static void
bpf_timed_out(void * arg,__unused void * dummy)1595 bpf_timed_out(void *arg, __unused void *dummy)
1596 {
1597 struct bpf_d *d = (struct bpf_d *)arg;
1598
1599 lck_mtx_lock(bpf_mlock);
1600 if (d->bd_state == BPF_WAITING) {
1601 /*
1602 * There's a select or kqueue waiting for this; if there's
1603 * now stuff to read, wake it up.
1604 */
1605 d->bd_state = BPF_TIMED_OUT;
1606 if (d->bd_slen != 0) {
1607 bpf_wakeup(d);
1608 }
1609 } else if (d->bd_state == BPF_DRAINING) {
1610 /*
1611 * A close is waiting for this to finish.
1612 * Mark it as finished, and wake the close up.
1613 */
1614 d->bd_state = BPF_IDLE;
1615 bpf_wakeup(d);
1616 }
1617 lck_mtx_unlock(bpf_mlock);
1618 }
1619
1620 /* keep in sync with bpf_movein above: */
1621 #define MAX_DATALINK_HDR_LEN (sizeof(struct firewire_header))
1622
1623 int
bpfwrite(dev_t dev,struct uio * uio,__unused int ioflag)1624 bpfwrite(dev_t dev, struct uio *uio, __unused int ioflag)
1625 {
1626 struct bpf_d *d;
1627 struct ifnet *ifp;
1628 mbuf_ref_t m = NULL;
1629 int error = 0;
1630 char dst_buf[SOCKADDR_HDR_LEN + MAX_DATALINK_HDR_LEN] = {};
1631 int bif_dlt;
1632 int bd_hdrcmplt;
1633 bpf_send_func bif_send;
1634
1635 lck_mtx_lock(bpf_mlock);
1636
1637 while (true) {
1638 d = bpf_dtab[minor(dev)];
1639 if (d == NULL || d == BPF_DEV_RESERVED ||
1640 (d->bd_flags & BPF_CLOSING) != 0) {
1641 lck_mtx_unlock(bpf_mlock);
1642 return ENXIO;
1643 }
1644
1645 if (d->bd_hbuf_write) {
1646 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfwrite",
1647 NULL);
1648 } else {
1649 break;
1650 }
1651 }
1652 d->bd_hbuf_write = true;
1653
1654 bpf_acquire_d(d);
1655
1656 ++d->bd_wcount;
1657
1658 if (d->bd_bif == NULL) {
1659 error = ENXIO;
1660 goto done;
1661 }
1662
1663 ifp = d->bd_bif->bif_ifp;
1664
1665 if (IFNET_IS_MANAGEMENT(ifp) &&
1666 IOCurrentTaskHasEntitlement(MANAGEMENT_DATA_ENTITLEMENT) == false) {
1667 ++d->bd_wdcount;
1668 bpf_release_d(d);
1669 lck_mtx_unlock(bpf_mlock);
1670 return ENETDOWN;
1671 }
1672
1673 if ((ifp->if_flags & IFF_UP) == 0) {
1674 error = ENETDOWN;
1675 goto done;
1676 }
1677 int resid = (int)uio_resid(uio);
1678 if (resid <= 0) {
1679 error = resid == 0 ? 0 : EINVAL;
1680 os_log(OS_LOG_DEFAULT, "bpfwrite: resid %d error %d", resid, error);
1681 goto done;
1682 }
1683 SA(dst_buf)->sa_len = sizeof(dst_buf);
1684
1685 /*
1686 * geting variables onto stack before dropping the lock
1687 */
1688 bif_dlt = (int)d->bd_bif->bif_dlt;
1689 bd_hdrcmplt = d->bd_hdrcmplt;
1690 bool batch_write = (d->bd_flags & BPF_BATCH_WRITE) ? true : false;
1691
1692 if (batch_write) {
1693 error = bpf_movein_batch(uio, d, &m, bd_hdrcmplt ? NULL : SA(dst_buf));
1694 if (error != 0) {
1695 goto done;
1696 }
1697 } else {
1698 error = bpf_movein(uio, resid, d, &m, bd_hdrcmplt ? NULL : SA(dst_buf));
1699 if (error != 0) {
1700 goto done;
1701 }
1702 bpf_set_packet_service_class(m, d->bd_traffic_class);
1703 }
1704
1705 /* verify the device is still open */
1706 if ((d->bd_flags & BPF_CLOSING) != 0) {
1707 error = ENXIO;
1708 goto done;
1709 }
1710
1711 if (d->bd_bif == NULL || d->bd_bif->bif_ifp != ifp) {
1712 error = ENXIO;
1713 goto done;
1714 }
1715
1716 bif_send = d->bd_bif->bif_send;
1717
1718 lck_mtx_unlock(bpf_mlock);
1719
1720 if (bd_hdrcmplt) {
1721 if (bif_send) {
1722 /*
1723 * Send one packet at a time, the driver frees the mbuf
1724 * but we need to take care of the leftover
1725 */
1726 while (m != NULL && error == 0) {
1727 struct mbuf *next = m->m_nextpkt;
1728
1729 m->m_nextpkt = NULL;
1730 error = bif_send(ifp, bif_dlt, m);
1731 m = next;
1732 }
1733 } else {
1734 error = dlil_output(ifp, 0, m, NULL, NULL,
1735 DLIL_OUTPUT_FLAGS_RAW, NULL);
1736 /* Make sure we do not double free */
1737 m = NULL;
1738 }
1739 } else {
1740 error = dlil_output(ifp, PF_INET, m, NULL, SA(dst_buf),
1741 DLIL_OUTPUT_FLAGS_NONE, NULL);
1742 /* Make sure we do not double free */
1743 m = NULL;
1744 }
1745
1746 lck_mtx_lock(bpf_mlock);
1747 done:
1748 if (error != 0 && m != NULL) {
1749 ++d->bd_wdcount;
1750 }
1751 if (m != NULL) {
1752 m_freem_list(m);
1753 }
1754 d->bd_hbuf_write = false;
1755 wakeup((caddr_t)d);
1756 bpf_release_d(d);
1757 lck_mtx_unlock(bpf_mlock);
1758
1759 return error;
1760 }
1761
1762 /*
1763 * Reset a descriptor by flushing its packet buffer and clearing the
1764 * receive and drop counts.
1765 */
1766 static void
reset_d(struct bpf_d * d)1767 reset_d(struct bpf_d *d)
1768 {
1769 if (d->bd_hbuf_read) {
1770 panic("resetting buffers during read");
1771 }
1772
1773 if (d->bd_hbuf) {
1774 /* Free the hold buffer. */
1775 d->bd_fbuf = d->bd_hbuf;
1776 d->bd_hbuf = NULL;
1777 }
1778 d->bd_slen = 0;
1779 d->bd_hlen = 0;
1780 d->bd_scnt = 0;
1781 d->bd_hcnt = 0;
1782 d->bd_rcount = 0;
1783 d->bd_dcount = 0;
1784 d->bd_fcount = 0;
1785 d->bd_wcount = 0;
1786 d->bd_wdcount = 0;
1787
1788 d->bd_prev_slen = 0;
1789 }
1790
1791 static struct bpf_d *
bpf_get_device_from_uuid(uuid_t uuid)1792 bpf_get_device_from_uuid(uuid_t uuid)
1793 {
1794 unsigned int i;
1795
1796 for (i = 0; i < nbpfilter; i++) {
1797 struct bpf_d *d = bpf_dtab[i];
1798
1799 if (d == NULL || d == BPF_DEV_RESERVED ||
1800 (d->bd_flags & BPF_CLOSING) != 0) {
1801 continue;
1802 }
1803 if (uuid_compare(uuid, d->bd_uuid) == 0) {
1804 return d;
1805 }
1806 }
1807
1808 return NULL;
1809 }
1810
1811 /*
1812 * The BIOCSETUP command "atomically" attach to the interface and
1813 * copy the buffer from another interface. This minimizes the risk
1814 * of missing packet because this is done while holding
1815 * the BPF global lock
1816 */
1817 static int
bpf_setup(struct bpf_d * d_to,uuid_t uuid_from,ifnet_t ifp,struct proc * proc)1818 bpf_setup(struct bpf_d *d_to, uuid_t uuid_from, ifnet_t ifp, struct proc *proc)
1819 {
1820 struct bpf_d *d_from;
1821 int error = 0;
1822
1823 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
1824
1825 /*
1826 * Sanity checks
1827 */
1828 d_from = bpf_get_device_from_uuid(uuid_from);
1829 if (d_from == NULL) {
1830 error = ENOENT;
1831 os_log_error(OS_LOG_DEFAULT,
1832 "%s: uuids not found error %d",
1833 __func__, error);
1834 return error;
1835 }
1836 if (d_from->bd_opened_by != d_to->bd_opened_by) {
1837 error = EACCES;
1838 os_log_error(OS_LOG_DEFAULT,
1839 "%s: processes not matching error %d",
1840 __func__, error);
1841 return error;
1842 }
1843
1844 /*
1845 * Prevent any read or write while copying
1846 */
1847 while (d_to->bd_hbuf_read || d_to->bd_hbuf_write) {
1848 msleep((caddr_t)d_to, bpf_mlock, PRINET, __func__, NULL);
1849 }
1850 d_to->bd_hbuf_read = true;
1851 d_to->bd_hbuf_write = true;
1852
1853 while (d_from->bd_hbuf_read || d_from->bd_hbuf_write) {
1854 msleep((caddr_t)d_from, bpf_mlock, PRINET, __func__, NULL);
1855 }
1856 d_from->bd_hbuf_read = true;
1857 d_from->bd_hbuf_write = true;
1858
1859 /*
1860 * Verify the devices have not been closed
1861 */
1862 if (d_to->bd_flags & BPF_CLOSING) {
1863 error = ENXIO;
1864 os_log_error(OS_LOG_DEFAULT,
1865 "%s: d_to is closing error %d",
1866 __func__, error);
1867 goto done;
1868 }
1869 if (d_from->bd_flags & BPF_CLOSING) {
1870 error = ENXIO;
1871 os_log_error(OS_LOG_DEFAULT,
1872 "%s: d_from is closing error %d",
1873 __func__, error);
1874 goto done;
1875 }
1876
1877 /*
1878 * For now require the same buffer size
1879 */
1880 if (d_from->bd_bufsize != d_to->bd_bufsize) {
1881 error = EINVAL;
1882 os_log_error(OS_LOG_DEFAULT,
1883 "%s: bufsizes not matching error %d",
1884 __func__, error);
1885 goto done;
1886 }
1887
1888 /*
1889 * Copy relevant options and flags
1890 */
1891 d_to->bd_flags = d_from->bd_flags & (BPF_EXTENDED_HDR | BPF_WANT_PKTAP |
1892 BPF_FINALIZE_PKTAP | BPF_TRUNCATE | BPF_PKTHDRV2 |
1893 BPF_COMP_REQ | BPF_COMP_ENABLED);
1894
1895 d_to->bd_headdrop = d_from->bd_headdrop;
1896
1897 /*
1898 * Allocate and copy the buffers
1899 */
1900 error = bpf_allocbufs(d_to);
1901 if (error != 0) {
1902 goto done;
1903 }
1904
1905 /*
1906 * Make sure the buffers are setup as expected by bpf_setif()
1907 */
1908 ASSERT(d_to->bd_hbuf == NULL);
1909 ASSERT(d_to->bd_sbuf != NULL);
1910 ASSERT(d_to->bd_fbuf != NULL);
1911
1912 /*
1913 * Copy the buffers and update the pointers and counts
1914 */
1915 memcpy(d_to->bd_sbuf, d_from->bd_sbuf, d_from->bd_slen);
1916 d_to->bd_slen = d_from->bd_slen;
1917 d_to->bd_scnt = d_from->bd_scnt;
1918
1919 if (d_from->bd_hbuf != NULL) {
1920 d_to->bd_hbuf = d_to->bd_fbuf;
1921 d_to->bd_fbuf = NULL;
1922 memcpy(d_to->bd_hbuf, d_from->bd_hbuf, d_from->bd_hlen);
1923 }
1924 d_to->bd_hlen = d_from->bd_hlen;
1925 d_to->bd_hcnt = d_from->bd_hcnt;
1926
1927 if (d_to->bd_flags & BPF_COMP_REQ) {
1928 ASSERT(d_to->bd_prev_sbuf != NULL);
1929 ASSERT(d_to->bd_prev_fbuf != NULL);
1930
1931 d_to->bd_prev_slen = d_from->bd_prev_slen;
1932 ASSERT(d_to->bd_prev_slen <= BPF_HDR_COMP_LEN_MAX);
1933 memcpy(d_to->bd_prev_sbuf, d_from->bd_prev_sbuf, BPF_HDR_COMP_LEN_MAX);
1934 }
1935
1936 d_to->bd_bcs = d_from->bd_bcs;
1937
1938 /*
1939 * Attach to the interface:
1940 * - don't reset the buffers
1941 * - we already prevent reads and writes
1942 * - the buffers are already allocated
1943 */
1944 error = bpf_setif(d_to, ifp, false, true, true, proc);
1945 if (error != 0) {
1946 os_log_error(OS_LOG_DEFAULT,
1947 "%s: bpf_setif() failed error %d",
1948 __func__, error);
1949 goto done;
1950 }
1951 done:
1952 d_from->bd_hbuf_read = false;
1953 d_from->bd_hbuf_write = false;
1954 wakeup((caddr_t)d_from);
1955
1956 d_to->bd_hbuf_read = false;
1957 d_to->bd_hbuf_write = false;
1958 wakeup((caddr_t)d_to);
1959
1960 return error;
1961 }
1962
1963 #if DEVELOPMENT || DEBUG
1964 #define BPF_IOC_LIST \
1965 X(FIONREAD) \
1966 X(SIOCGIFADDR) \
1967 X(BIOCGBLEN) \
1968 X(BIOCSBLEN) \
1969 X(BIOCSETF32) \
1970 X(BIOCSETFNR32) \
1971 X(BIOCSETF64) \
1972 X(BIOCSETFNR64) \
1973 X(BIOCFLUSH) \
1974 X(BIOCPROMISC) \
1975 X(BIOCGDLT) \
1976 X(BIOCGDLTLIST) \
1977 X(BIOCSDLT) \
1978 X(BIOCGETIF) \
1979 X(BIOCSETIF) \
1980 X(BIOCSRTIMEOUT32) \
1981 X(BIOCSRTIMEOUT64) \
1982 X(BIOCGRTIMEOUT32) \
1983 X(BIOCGRTIMEOUT64) \
1984 X(BIOCGSTATS) \
1985 X(BIOCIMMEDIATE) \
1986 X(BIOCVERSION) \
1987 X(BIOCGHDRCMPLT) \
1988 X(BIOCSHDRCMPLT) \
1989 X(BIOCGSEESENT) \
1990 X(BIOCSSEESENT) \
1991 X(BIOCSETTC) \
1992 X(BIOCGETTC) \
1993 X(FIONBIO) \
1994 X(FIOASYNC) \
1995 X(BIOCSRSIG) \
1996 X(BIOCGRSIG) \
1997 X(BIOCSEXTHDR) \
1998 X(BIOCGIFATTACHCOUNT) \
1999 X(BIOCGWANTPKTAP) \
2000 X(BIOCSWANTPKTAP) \
2001 X(BIOCSHEADDROP) \
2002 X(BIOCGHEADDROP) \
2003 X(BIOCSTRUNCATE) \
2004 X(BIOCGETUUID) \
2005 X(BIOCSETUP) \
2006 X(BIOCSPKTHDRV2) \
2007 X(BIOCGHDRCOMP) \
2008 X(BIOCSHDRCOMP) \
2009 X(BIOCGHDRCOMPSTATS) \
2010 X(BIOCGHDRCOMPON) \
2011 X(BIOCGDIRECTION) \
2012 X(BIOCSDIRECTION) \
2013 X(BIOCSWRITEMAX) \
2014 X(BIOCGWRITEMAX) \
2015 X(BIOCGBATCHWRITE) \
2016 X(BIOCSBATCHWRITE) \
2017 X(BIOCGNOTSTAMP) \
2018 X(BIOCSNOTSTAMP)
2019
2020 static void
log_bpf_ioctl_str(struct bpf_d * d,u_long cmd,int error)2021 log_bpf_ioctl_str(struct bpf_d *d, u_long cmd, int error)
2022 {
2023 const char *p = NULL;
2024 char str[32];
2025
2026 #define X(x) case x: { p = #x ; printf("%s\n", p); break; }
2027 switch (cmd) {
2028 BPF_IOC_LIST
2029 }
2030 #undef X
2031 if (p == NULL) {
2032 snprintf(str, sizeof(str), "0x%08x", (unsigned int)cmd);
2033 p = str;
2034 }
2035 os_log(OS_LOG_DEFAULT, "bpfioctl bpf%u %s error: %d",
2036 d->bd_dev_minor, p, error);
2037 }
2038 #endif /* DEVELOPMENT || DEBUG */
2039
2040 /*
2041 * FIONREAD Check for read packet available.
2042 * SIOCGIFADDR Get interface address - convenient hook to driver.
2043 * BIOCGBLEN Get buffer len [for read()].
2044 * BIOCSETF Set ethernet read filter.
2045 * BIOCFLUSH Flush read packet buffer.
2046 * BIOCPROMISC Put interface into promiscuous mode.
2047 * BIOCGDLT Get link layer type.
2048 * BIOCGETIF Get interface name.
2049 * BIOCSETIF Set interface.
2050 * BIOCSRTIMEOUT Set read timeout.
2051 * BIOCGRTIMEOUT Get read timeout.
2052 * BIOCGSTATS Get packet stats.
2053 * BIOCIMMEDIATE Set immediate mode.
2054 * BIOCVERSION Get filter language version.
2055 * BIOCGHDRCMPLT Get "header already complete" flag
2056 * BIOCSHDRCMPLT Set "header already complete" flag
2057 * BIOCGSEESENT Get "see packets sent" flag
2058 * BIOCSSEESENT Set "see packets sent" flag
2059 * BIOCSETTC Set traffic class.
2060 * BIOCGETTC Get traffic class.
2061 * BIOCSEXTHDR Set "extended header" flag
2062 * BIOCSHEADDROP Drop head of the buffer if user is not reading
2063 * BIOCGHEADDROP Get "head-drop" flag
2064 */
2065 /* ARGSUSED */
2066 int
bpfioctl(dev_t dev,u_long cmd,caddr_t __sized_by (IOCPARM_LEN (cmd))addr,__unused int flags,struct proc * proc)2067 bpfioctl(dev_t dev, u_long cmd, caddr_t __sized_by(IOCPARM_LEN(cmd)) addr,
2068 __unused int flags, struct proc *proc)
2069 {
2070 struct bpf_d *d;
2071 int error = 0;
2072 u_int int_arg;
2073 struct ifreq ifr = {};
2074
2075 lck_mtx_lock(bpf_mlock);
2076 d = bpf_dtab[minor(dev)];
2077 if (d == NULL || d == BPF_DEV_RESERVED ||
2078 (d->bd_flags & BPF_CLOSING) != 0) {
2079 lck_mtx_unlock(bpf_mlock);
2080 return ENXIO;
2081 }
2082
2083 bpf_acquire_d(d);
2084
2085 if (d->bd_state == BPF_WAITING) {
2086 bpf_stop_timer(d);
2087 }
2088 d->bd_state = BPF_IDLE;
2089
2090 switch (cmd) {
2091 default:
2092 error = EINVAL;
2093 break;
2094
2095 /*
2096 * Check for read packet available.
2097 */
2098 case FIONREAD: { /* int */
2099 int n;
2100
2101 n = d->bd_slen;
2102 if (d->bd_hbuf && d->bd_hbuf_read) {
2103 n += d->bd_hlen;
2104 }
2105
2106 bcopy(&n, addr, sizeof(n));
2107 break;
2108 }
2109
2110 case SIOCGIFADDR: { /* struct ifreq */
2111 struct ifnet *ifp;
2112
2113 if (d->bd_bif == 0) {
2114 error = EINVAL;
2115 } else {
2116 ifp = d->bd_bif->bif_ifp;
2117 error = ifnet_ioctl(ifp, 0, cmd, addr);
2118 }
2119 break;
2120 }
2121
2122 /*
2123 * Get buffer len [for read()].
2124 */
2125 case BIOCGBLEN: { /* u_int */
2126 static_assert(sizeof(d->bd_bufsize) == sizeof(u_int));
2127 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
2128 break;
2129 }
2130
2131 /*
2132 * Set buffer length.
2133 */
2134 case BIOCSBLEN: { /* u_int */
2135 u_int size;
2136
2137 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
2138 /*
2139 * Interface already attached, unable to change buffers
2140 */
2141 error = EINVAL;
2142 break;
2143 }
2144 bcopy(addr, &size, sizeof(size));
2145
2146 if (size > BPF_BUFSIZE_CAP) {
2147 d->bd_bufsize = BPF_BUFSIZE_CAP;
2148
2149 os_log_info(OS_LOG_DEFAULT,
2150 "bpf%d BIOCSBLEN capped to %u from %u",
2151 minor(dev), d->bd_bufsize, size);
2152 } else if (size < BPF_MINBUFSIZE) {
2153 d->bd_bufsize = BPF_MINBUFSIZE;
2154
2155 os_log_info(OS_LOG_DEFAULT,
2156 "bpf%d BIOCSBLEN bumped to %u from %u",
2157 minor(dev), d->bd_bufsize, size);
2158 } else {
2159 d->bd_bufsize = size;
2160
2161 os_log_info(OS_LOG_DEFAULT,
2162 "bpf%d BIOCSBLEN %u",
2163 minor(dev), d->bd_bufsize);
2164 }
2165
2166 /* It's a read/write ioctl */
2167 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
2168 break;
2169 }
2170 /*
2171 * Set link layer read filter.
2172 */
2173 case BIOCSETF32:
2174 case BIOCSETFNR32: { /* struct bpf_program32 */
2175 struct bpf_program32 prg32;
2176
2177 bcopy(addr, &prg32, sizeof(prg32));
2178 error = bpf_setf(d, prg32.bf_len,
2179 CAST_USER_ADDR_T(prg32.bf_insns), cmd);
2180 break;
2181 }
2182
2183 case BIOCSETF64:
2184 case BIOCSETFNR64: { /* struct bpf_program64 */
2185 struct bpf_program64 prg64;
2186
2187 bcopy(addr, &prg64, sizeof(prg64));
2188 error = bpf_setf(d, prg64.bf_len, CAST_USER_ADDR_T(prg64.bf_insns), cmd);
2189 break;
2190 }
2191
2192 /*
2193 * Flush read packet buffer.
2194 */
2195 case BIOCFLUSH:
2196 while (d->bd_hbuf_read) {
2197 msleep((caddr_t)d, bpf_mlock, PRINET, "BIOCFLUSH",
2198 NULL);
2199 }
2200 if ((d->bd_flags & BPF_CLOSING) != 0) {
2201 error = ENXIO;
2202 break;
2203 }
2204 reset_d(d);
2205 break;
2206
2207 /*
2208 * Put interface into promiscuous mode.
2209 */
2210 case BIOCPROMISC:
2211 if (d->bd_bif == 0) {
2212 /*
2213 * No interface attached yet.
2214 */
2215 error = EINVAL;
2216 break;
2217 }
2218 if (d->bd_promisc == 0) {
2219 lck_mtx_unlock(bpf_mlock);
2220 error = ifnet_set_promiscuous(d->bd_bif->bif_ifp, 1);
2221 lck_mtx_lock(bpf_mlock);
2222 if (error == 0) {
2223 d->bd_promisc = 1;
2224 }
2225 }
2226 break;
2227
2228 /*
2229 * Get device parameters.
2230 */
2231 case BIOCGDLT: { /* u_int */
2232 if (d->bd_bif == 0) {
2233 error = EINVAL;
2234 } else {
2235 static_assert(sizeof(d->bd_bif->bif_dlt) == sizeof(u_int));
2236 bcopy(&d->bd_bif->bif_dlt, addr, sizeof(u_int));
2237 }
2238 break;
2239 }
2240
2241 /*
2242 * Get a list of supported data link types.
2243 */
2244 case BIOCGDLTLIST: /* struct bpf_dltlist */
2245 if (d->bd_bif == NULL) {
2246 error = EINVAL;
2247 } else {
2248 error = bpf_getdltlist(d, addr, proc);
2249 }
2250 break;
2251
2252 /*
2253 * Set data link type.
2254 */
2255 case BIOCSDLT: /* u_int */
2256 if (d->bd_bif == NULL) {
2257 error = EINVAL;
2258 } else {
2259 u_int dlt;
2260
2261 bcopy(addr, &dlt, sizeof(dlt));
2262
2263 if (dlt == DLT_PKTAP &&
2264 !(d->bd_flags & BPF_WANT_PKTAP)) {
2265 dlt = DLT_RAW;
2266 }
2267 error = bpf_setdlt(d, dlt, proc);
2268 }
2269 break;
2270
2271 /*
2272 * Get interface name.
2273 */
2274 case BIOCGETIF: /* struct ifreq */
2275 if (d->bd_bif == 0) {
2276 error = EINVAL;
2277 } else {
2278 struct ifnet *const ifp = d->bd_bif->bif_ifp;
2279
2280 snprintf(((struct ifreq *)(void *)addr)->ifr_name,
2281 sizeof(ifr.ifr_name), "%s", if_name(ifp));
2282 }
2283 break;
2284
2285 /*
2286 * Set interface.
2287 */
2288 case BIOCSETIF: { /* struct ifreq */
2289 ifnet_t ifp;
2290
2291 bcopy(addr, &ifr, sizeof(ifr));
2292 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2293 ifp = ifunit(__unsafe_null_terminated_from_indexable(ifr.ifr_name,
2294 &ifr.ifr_name[IFNAMSIZ - 1]));
2295 if (ifp == NULL) {
2296 error = ENXIO;
2297 } else {
2298 error = bpf_setif(d, ifp, true, false, false, proc);
2299 }
2300 break;
2301 }
2302
2303 /*
2304 * Set read timeout.
2305 */
2306 case BIOCSRTIMEOUT32: { /* struct user32_timeval */
2307 struct user32_timeval _tv;
2308 struct timeval tv;
2309
2310 bcopy(addr, &_tv, sizeof(_tv));
2311 tv.tv_sec = _tv.tv_sec;
2312 tv.tv_usec = _tv.tv_usec;
2313
2314 /*
2315 * Subtract 1 tick from tvtohz() since this isn't
2316 * a one-shot timer.
2317 */
2318 if ((error = itimerfix(&tv)) == 0) {
2319 d->bd_rtout = tvtohz(&tv) - 1;
2320 }
2321 break;
2322 }
2323
2324 case BIOCSRTIMEOUT64: { /* struct user64_timeval */
2325 struct user64_timeval _tv;
2326 struct timeval tv;
2327
2328 bcopy(addr, &_tv, sizeof(_tv));
2329 tv.tv_sec = (__darwin_time_t)_tv.tv_sec;
2330 tv.tv_usec = _tv.tv_usec;
2331
2332 /*
2333 * Subtract 1 tick from tvtohz() since this isn't
2334 * a one-shot timer.
2335 */
2336 if ((error = itimerfix(&tv)) == 0) {
2337 d->bd_rtout = tvtohz(&tv) - 1;
2338 }
2339 break;
2340 }
2341
2342 /*
2343 * Get read timeout.
2344 */
2345 case BIOCGRTIMEOUT32: { /* struct user32_timeval */
2346 struct user32_timeval tv;
2347
2348 bzero(&tv, sizeof(tv));
2349 tv.tv_sec = d->bd_rtout / hz;
2350 tv.tv_usec = (d->bd_rtout % hz) * tick;
2351 bcopy(&tv, addr, sizeof(tv));
2352 break;
2353 }
2354
2355 case BIOCGRTIMEOUT64: { /* struct user64_timeval */
2356 struct user64_timeval tv;
2357
2358 bzero(&tv, sizeof(tv));
2359 tv.tv_sec = d->bd_rtout / hz;
2360 tv.tv_usec = (d->bd_rtout % hz) * tick;
2361 bcopy(&tv, addr, sizeof(tv));
2362 break;
2363 }
2364
2365 /*
2366 * Get packet stats.
2367 */
2368 case BIOCGSTATS: { /* struct bpf_stat */
2369 struct bpf_stat bs;
2370
2371 bzero(&bs, sizeof(bs));
2372 bs.bs_recv = (u_int)d->bd_rcount;
2373 bs.bs_drop = (u_int)d->bd_dcount;
2374 bcopy(&bs, addr, sizeof(bs));
2375 break;
2376 }
2377
2378 /*
2379 * Set immediate mode.
2380 */
2381 case BIOCIMMEDIATE: /* u_int */
2382 d->bd_immediate = *(u_char *)(void *)addr;
2383 break;
2384
2385 case BIOCVERSION: { /* struct bpf_version */
2386 struct bpf_version bv;
2387
2388 bzero(&bv, sizeof(bv));
2389 bv.bv_major = BPF_MAJOR_VERSION;
2390 bv.bv_minor = BPF_MINOR_VERSION;
2391 bcopy(&bv, addr, sizeof(bv));
2392 break;
2393 }
2394
2395 /*
2396 * Get "header already complete" flag
2397 */
2398 case BIOCGHDRCMPLT: { /* u_int */
2399 u_int *addr_int = (u_int *)(void *)addr;
2400 *addr_int = d->bd_hdrcmplt;
2401 break;
2402 }
2403
2404 /*
2405 * Set "header already complete" flag
2406 */
2407 case BIOCSHDRCMPLT: /* u_int */
2408 bcopy(addr, &int_arg, sizeof(int_arg));
2409 if (int_arg == 0 && (d->bd_flags & BPF_BATCH_WRITE)) {
2410 os_log(OS_LOG_DEFAULT,
2411 "bpf%u cannot set BIOCSHDRCMPLT when BIOCSBATCHWRITE is set",
2412 d->bd_dev_minor);
2413 error = EINVAL;
2414 break;
2415 }
2416 d->bd_hdrcmplt = int_arg ? 1 : 0;
2417 break;
2418
2419 /*
2420 * Get "see sent packets" flag
2421 */
2422 case BIOCGSEESENT: { /* u_int */
2423 int_arg = 0;
2424
2425 if (d->bd_direction & BPF_D_OUT) {
2426 int_arg = 1;
2427 }
2428 bcopy(&int_arg, addr, sizeof(u_int));
2429 break;
2430 }
2431 /*
2432 * Set "see sent packets" flag
2433 */
2434 case BIOCSSEESENT: { /* u_int */
2435 bcopy(addr, &int_arg, sizeof(u_int));
2436
2437 if (int_arg == 0) {
2438 d->bd_direction = BPF_D_IN;
2439 } else {
2440 d->bd_direction = BPF_D_INOUT;
2441 }
2442 break;
2443 }
2444 /*
2445 * Get direction of tapped packets that can be seen for reading
2446 */
2447 case BIOCGDIRECTION: { /* u_int */
2448 int_arg = d->bd_direction;
2449
2450 bcopy(&int_arg, addr, sizeof(u_int));
2451 break;
2452 }
2453 /*
2454 * Set direction of tapped packets that can be seen for reading
2455 */
2456 case BIOCSDIRECTION: { /* u_int */
2457 bcopy(addr, &int_arg, sizeof(u_int));
2458
2459 switch (int_arg) {
2460 case BPF_D_NONE:
2461 case BPF_D_IN:
2462 case BPF_D_OUT:
2463 case BPF_D_INOUT:
2464 d->bd_direction = int_arg;
2465 break;
2466 default:
2467 error = EINVAL;
2468 break;
2469 }
2470 break;
2471 }
2472 /*
2473 * Set traffic service class
2474 */
2475 case BIOCSETTC: { /* int */
2476 int tc;
2477
2478 bcopy(addr, &tc, sizeof(int));
2479 if (tc != 0 && (d->bd_flags & BPF_BATCH_WRITE)) {
2480 os_log(OS_LOG_DEFAULT,
2481 "bpf%u cannot set BIOCSETTC when BIOCSBATCHWRITE is set",
2482 d->bd_dev_minor);
2483 error = EINVAL;
2484 break;
2485 }
2486 error = bpf_set_traffic_class(d, tc);
2487 break;
2488 }
2489
2490 /*
2491 * Get traffic service class
2492 */
2493 case BIOCGETTC: { /* int */
2494 static_assert(sizeof(d->bd_traffic_class) == sizeof(int));
2495 bcopy(&d->bd_traffic_class, addr, sizeof(int));
2496 break;
2497 }
2498
2499 case FIONBIO: /* Non-blocking I/O; int */
2500 break;
2501
2502 case FIOASYNC: { /* Send signal on receive packets; int */
2503 static_assert(sizeof(d->bd_async) == sizeof(int));
2504 bcopy(addr, &d->bd_async, sizeof(int));
2505 break;
2506 }
2507
2508 case BIOCSRSIG: { /* Set receive signal; u_int */
2509 u_int sig;
2510
2511 bcopy(addr, &sig, sizeof(u_int));
2512
2513 if (sig >= NSIG) {
2514 error = EINVAL;
2515 } else {
2516 d->bd_sig = sig;
2517 }
2518 break;
2519 }
2520 case BIOCGRSIG: { /* u_int */
2521 static_assert(sizeof(d->bd_sig) == sizeof(u_int));
2522 bcopy(&d->bd_sig, addr, sizeof(u_int));
2523 break;
2524 }
2525
2526 case BIOCSEXTHDR: /* u_int */
2527 bcopy(addr, &int_arg, sizeof(int_arg));
2528 if (int_arg) {
2529 d->bd_flags |= BPF_EXTENDED_HDR;
2530 } else {
2531 d->bd_flags &= ~BPF_EXTENDED_HDR;
2532 }
2533 break;
2534
2535 case BIOCGIFATTACHCOUNT: { /* struct ifreq */
2536 ifnet_t ifp;
2537 struct bpf_if *bp;
2538
2539 bcopy(addr, &ifr, sizeof(ifr));
2540 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2541 ifp = ifunit(__unsafe_null_terminated_from_indexable(ifr.ifr_name,
2542 &ifr.ifr_name[IFNAMSIZ - 1]));
2543 if (ifp == NULL) {
2544 error = ENXIO;
2545 break;
2546 }
2547 ifr.ifr_intval = 0;
2548 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2549 struct bpf_d *bpf_d;
2550
2551 if (bp->bif_ifp == NULL || bp->bif_ifp != ifp) {
2552 continue;
2553 }
2554 for (bpf_d = bp->bif_dlist; bpf_d;
2555 bpf_d = bpf_d->bd_next) {
2556 ifr.ifr_intval += 1;
2557 }
2558 }
2559 bcopy(&ifr, addr, sizeof(ifr));
2560 break;
2561 }
2562 case BIOCGWANTPKTAP: /* u_int */
2563 int_arg = d->bd_flags & BPF_WANT_PKTAP ? 1 : 0;
2564 bcopy(&int_arg, addr, sizeof(int_arg));
2565 break;
2566
2567 case BIOCSWANTPKTAP: /* u_int */
2568 bcopy(addr, &int_arg, sizeof(int_arg));
2569 if (int_arg) {
2570 d->bd_flags |= BPF_WANT_PKTAP;
2571 } else {
2572 d->bd_flags &= ~BPF_WANT_PKTAP;
2573 }
2574 break;
2575
2576 case BIOCSHEADDROP:
2577 bcopy(addr, &int_arg, sizeof(int_arg));
2578 d->bd_headdrop = int_arg ? 1 : 0;
2579 break;
2580
2581 case BIOCGHEADDROP: {
2582 u_int *addr_int = (uint *)(void *)addr;
2583 *addr_int = d->bd_headdrop;
2584 break;
2585 }
2586
2587 case BIOCSTRUNCATE:
2588 bcopy(addr, &int_arg, sizeof(int_arg));
2589 if (int_arg) {
2590 d->bd_flags |= BPF_TRUNCATE;
2591 } else {
2592 d->bd_flags &= ~BPF_TRUNCATE;
2593 }
2594 break;
2595
2596 case BIOCGETUUID: /* uuid_t */
2597 bcopy(&d->bd_uuid, addr, sizeof(uuid_t));
2598 break;
2599
2600 case BIOCSETUP: {
2601 struct bpf_setup_args bsa;
2602 ifnet_t ifp;
2603
2604 bcopy(addr, &bsa, sizeof(struct bpf_setup_args));
2605 bsa.bsa_ifname[IFNAMSIZ - 1] = 0;
2606 ifp = ifunit(__unsafe_null_terminated_from_indexable(bsa.bsa_ifname,
2607 &bsa.bsa_ifname[IFNAMSIZ - 1]));
2608 if (ifp == NULL) {
2609 error = ENXIO;
2610 os_log_error(OS_LOG_DEFAULT,
2611 "%s: ifnet not found for %s error %d",
2612 __func__, bsa.bsa_ifname, error);
2613 break;
2614 }
2615
2616 error = bpf_setup(d, bsa.bsa_uuid, ifp, proc);
2617 break;
2618 }
2619 case BIOCSPKTHDRV2:
2620 bcopy(addr, &int_arg, sizeof(int_arg));
2621 if (int_arg != 0) {
2622 d->bd_flags |= BPF_PKTHDRV2;
2623 } else {
2624 d->bd_flags &= ~BPF_PKTHDRV2;
2625 }
2626 break;
2627
2628 case BIOCGPKTHDRV2:
2629 int_arg = d->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
2630 bcopy(&int_arg, addr, sizeof(int_arg));
2631 break;
2632
2633 case BIOCGHDRCOMP:
2634 int_arg = d->bd_flags & BPF_COMP_REQ ? 1 : 0;
2635 bcopy(&int_arg, addr, sizeof(int_arg));
2636 break;
2637
2638 case BIOCSHDRCOMP:
2639 bcopy(addr, &int_arg, sizeof(int_arg));
2640 if (int_arg != 0 && int_arg != 1) {
2641 return EINVAL;
2642 }
2643 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
2644 /*
2645 * Interface already attached, unable to change buffers
2646 */
2647 error = EINVAL;
2648 break;
2649 }
2650 if (int_arg != 0) {
2651 d->bd_flags |= BPF_COMP_REQ;
2652 if (bpf_hdr_comp_enable != 0) {
2653 d->bd_flags |= BPF_COMP_ENABLED;
2654 }
2655 } else {
2656 d->bd_flags &= ~(BPF_COMP_REQ | BPF_COMP_ENABLED);
2657 }
2658 break;
2659
2660 case BIOCGHDRCOMPON:
2661 int_arg = d->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
2662 bcopy(&int_arg, addr, sizeof(int_arg));
2663 break;
2664
2665 case BIOCGHDRCOMPSTATS: {
2666 struct bpf_comp_stats bcs = {};
2667
2668 bcs = d->bd_bcs;
2669
2670 bcopy(&bcs, addr, sizeof(bcs));
2671 break;
2672 }
2673 case BIOCSWRITEMAX:
2674 bcopy(addr, &int_arg, sizeof(int_arg));
2675 if (int_arg > BPF_WRITE_MAX) {
2676 os_log(OS_LOG_DEFAULT, "bpf%u bd_write_size_max %u too big",
2677 d->bd_dev_minor, d->bd_write_size_max);
2678 error = EINVAL;
2679 break;
2680 }
2681 d->bd_write_size_max = int_arg;
2682 break;
2683
2684 case BIOCGWRITEMAX:
2685 int_arg = d->bd_write_size_max;
2686 bcopy(&int_arg, addr, sizeof(int_arg));
2687 break;
2688
2689 case BIOCGBATCHWRITE: /* int */
2690 int_arg = d->bd_flags & BPF_BATCH_WRITE ? 1 : 0;
2691 bcopy(&int_arg, addr, sizeof(int_arg));
2692 break;
2693
2694 case BIOCSBATCHWRITE: /* int */
2695 bcopy(addr, &int_arg, sizeof(int_arg));
2696 if (int_arg != 0) {
2697 if (d->bd_hdrcmplt == 0) {
2698 os_log(OS_LOG_DEFAULT,
2699 "bpf%u cannot set BIOCSBATCHWRITE when BIOCSHDRCMPLT is not set",
2700 d->bd_dev_minor);
2701 error = EINVAL;
2702 break;
2703 }
2704 if (d->bd_traffic_class != 0) {
2705 os_log(OS_LOG_DEFAULT,
2706 "bpf%u cannot set BIOCSBATCHWRITE when BIOCSETTC is set",
2707 d->bd_dev_minor);
2708 error = EINVAL;
2709 break;
2710 }
2711 d->bd_flags |= BPF_BATCH_WRITE;
2712 } else {
2713 d->bd_flags &= ~BPF_BATCH_WRITE;
2714 }
2715 break;
2716 case BIOCGNOTSTAMP:
2717 if (d->bd_tstamp == BPF_T_NONE) {
2718 int_arg = 1;
2719 } else {
2720 int_arg = 0;
2721 }
2722 bcopy(&int_arg, addr, sizeof(int_arg));
2723 break;
2724 case BIOCSNOTSTAMP:
2725 bcopy(addr, &int_arg, sizeof(int_arg));
2726 switch (int_arg) {
2727 case 0:
2728 d->bd_tstamp = BPF_T_MICROTIME;
2729 break;
2730 default:
2731 d->bd_tstamp = BPF_T_NONE;
2732 break;
2733 }
2734 break;
2735 case BIOCGDVRTIN:
2736 int_arg = d->bd_flags & BPF_DIVERT_IN ? 1 : 0;
2737 bcopy(&int_arg, addr, sizeof(int_arg));
2738 break;
2739 case BIOCSDVRTIN:
2740 if (d->bd_bif == NULL) {
2741 /*
2742 * No interface attached yet.
2743 */
2744 error = ENXIO;
2745 break;
2746 }
2747 bcopy(addr, &int_arg, sizeof(int_arg));
2748 if (int_arg == 0) {
2749 if ((d->bd_flags & BPF_DIVERT_IN) == 0) {
2750 error = EINVAL;
2751 break;
2752 }
2753 d->bd_flags &= ~BPF_DIVERT_IN;
2754 if_clear_xflags(d->bd_bif->bif_ifp, IFXF_DISABLE_INPUT);
2755 } else {
2756 if ((d->bd_flags & BPF_DIVERT_IN) != 0 ||
2757 (d->bd_bif->bif_ifp->if_xflags & IFXF_DISABLE_INPUT) != 0) {
2758 error = EALREADY;
2759 break;
2760 }
2761 d->bd_flags |= BPF_DIVERT_IN;
2762 if_set_xflags(d->bd_bif->bif_ifp, IFXF_DISABLE_INPUT);
2763 }
2764 break;
2765 }
2766
2767 #if DEVELOPMENT || DEBUG
2768 if (bpf_debug > 0) {
2769 log_bpf_ioctl_str(d, cmd, error);
2770 }
2771 #endif /* DEVELOPMENT || DEBUG */
2772
2773
2774 bpf_release_d(d);
2775 lck_mtx_unlock(bpf_mlock);
2776
2777 return error;
2778 }
2779
2780 /*
2781 * Set d's packet filter program to fp. If this file already has a filter,
2782 * free it and replace it. Returns EINVAL for bogus requests.
2783 */
2784 static int
bpf_setf(struct bpf_d * d,u_int bf_len,user_addr_t bf_insns,u_long cmd)2785 bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns,
2786 u_long cmd)
2787 {
2788 struct bpf_insn *fcode, *old;
2789 u_int flen, size;
2790
2791 while (d->bd_hbuf_read) {
2792 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setf", NULL);
2793 }
2794
2795 if ((d->bd_flags & BPF_CLOSING) != 0) {
2796 return ENXIO;
2797 }
2798
2799 old = d->bd_filter;
2800 if (bf_insns == USER_ADDR_NULL) {
2801 if (bf_len != 0) {
2802 return EINVAL;
2803 }
2804 d->bd_filter = NULL;
2805 d->bd_filter_len = 0;
2806 reset_d(d);
2807 if (old != 0) {
2808 kfree_data_addr(old);
2809 }
2810 return 0;
2811 }
2812 flen = bf_len;
2813 if (flen > BPF_MAXINSNS) {
2814 return EINVAL;
2815 }
2816
2817 size = flen * sizeof(struct bpf_insn);
2818 fcode = (struct bpf_insn *) kalloc_data(size, Z_WAITOK | Z_ZERO);
2819 if (fcode == NULL) {
2820 return ENOMEM;
2821 }
2822 if (copyin(bf_insns, (caddr_t)fcode, size) == 0 &&
2823 bpf_validate(fcode, (int)flen)) {
2824 d->bd_filter = fcode;
2825 d->bd_filter_len = flen;
2826
2827 if (cmd == BIOCSETF32 || cmd == BIOCSETF64) {
2828 reset_d(d);
2829 }
2830
2831 if (old != 0) {
2832 kfree_data_addr(old);
2833 }
2834
2835 return 0;
2836 }
2837 kfree_data(fcode, size);
2838 return EINVAL;
2839 }
2840
2841 /*
2842 * Detach a file from its current interface (if attached at all) and attach
2843 * to the interface indicated by the name stored in ifr.
2844 * Return an errno or 0.
2845 */
2846 static int
bpf_setif(struct bpf_d * d,ifnet_t theywant,bool do_reset,bool has_hbuf_read_write,bool has_bufs_allocated,struct proc * proc)2847 bpf_setif(struct bpf_d *d, ifnet_t theywant, bool do_reset, bool has_hbuf_read_write,
2848 bool has_bufs_allocated, struct proc *proc)
2849 {
2850 struct bpf_if *bp;
2851 int error;
2852
2853 while (!has_hbuf_read_write && (d->bd_hbuf_read || d->bd_hbuf_write)) {
2854 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setif", NULL);
2855 }
2856
2857 if ((d->bd_flags & BPF_CLOSING) != 0) {
2858 return ENXIO;
2859 }
2860
2861 /*
2862 * Look through attached interfaces for the named one.
2863 */
2864 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2865 struct ifnet *ifp = bp->bif_ifp;
2866
2867 if (ifp == 0 || ifp != theywant) {
2868 continue;
2869 }
2870 /*
2871 * Do not use DLT_PKTAP, unless requested explicitly
2872 */
2873 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2874 continue;
2875 }
2876 /*
2877 * Skip the coprocessor interface
2878 */
2879 if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) {
2880 continue;
2881 }
2882 /*
2883 * We found the requested interface.
2884 * Allocate the packet buffers.
2885 */
2886 if (has_bufs_allocated == false) {
2887 error = bpf_allocbufs(d);
2888 if (error != 0) {
2889 return error;
2890 }
2891 }
2892 /*
2893 * Detach if attached to something else.
2894 */
2895 if (bp != d->bd_bif) {
2896 if (d->bd_bif != NULL) {
2897 if (bpf_detachd(d, proc) != 0) {
2898 return ENXIO;
2899 }
2900 }
2901 if (bpf_attachd(d, bp) != 0) {
2902 return ENXIO;
2903 }
2904 }
2905 if (do_reset) {
2906 reset_d(d);
2907 }
2908 os_log(OS_LOG_DEFAULT, "bpf%u attached to %s by %s:%u",
2909 d->bd_dev_minor, if_name(theywant), proc_name_address(proc), proc_pid(proc));
2910 return 0;
2911 }
2912 /* Not found. */
2913 return ENXIO;
2914 }
2915
2916 /*
2917 * Get a list of available data link type of the interface.
2918 */
2919 static int
bpf_getdltlist(struct bpf_d * d,caddr_t __bidi_indexable addr,struct proc * proc)2920 bpf_getdltlist(struct bpf_d *d, caddr_t __bidi_indexable addr, struct proc *proc)
2921 {
2922 u_int n;
2923 int error;
2924 struct ifnet *ifp;
2925 struct bpf_if *bp;
2926 user_addr_t dlist;
2927 struct bpf_dltlist bfl;
2928
2929 bcopy(addr, &bfl, sizeof(bfl));
2930 if (proc_is64bit(proc)) {
2931 dlist = (user_addr_t)bfl.bfl_u.bflu_pad;
2932 } else {
2933 dlist = CAST_USER_ADDR_T(bfl.bfl_u.bflu_list);
2934 }
2935
2936 ifp = d->bd_bif->bif_ifp;
2937 n = 0;
2938 error = 0;
2939
2940 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2941 if (bp->bif_ifp != ifp) {
2942 continue;
2943 }
2944 /*
2945 * Do not use DLT_PKTAP, unless requested explicitly
2946 */
2947 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2948 continue;
2949 }
2950 if (dlist != USER_ADDR_NULL) {
2951 if (n >= bfl.bfl_len) {
2952 return ENOMEM;
2953 }
2954 error = copyout(&bp->bif_dlt, dlist,
2955 sizeof(bp->bif_dlt));
2956 if (error != 0) {
2957 break;
2958 }
2959 dlist += sizeof(bp->bif_dlt);
2960 }
2961 n++;
2962 }
2963 bfl.bfl_len = n;
2964 bcopy(&bfl, addr, sizeof(bfl));
2965
2966 return error;
2967 }
2968
2969 /*
2970 * Set the data link type of a BPF instance.
2971 */
2972 static int
bpf_setdlt(struct bpf_d * d,uint32_t dlt,struct proc * proc)2973 bpf_setdlt(struct bpf_d *d, uint32_t dlt, struct proc *proc)
2974 {
2975 int error, opromisc;
2976 struct ifnet *ifp;
2977 struct bpf_if *bp;
2978
2979 if (d->bd_bif->bif_dlt == dlt) {
2980 return 0;
2981 }
2982
2983 while (d->bd_hbuf_read) {
2984 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setdlt", NULL);
2985 }
2986
2987 if ((d->bd_flags & BPF_CLOSING) != 0) {
2988 return ENXIO;
2989 }
2990
2991 ifp = d->bd_bif->bif_ifp;
2992 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2993 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) {
2994 /*
2995 * Do not use DLT_PKTAP, unless requested explicitly
2996 */
2997 if (bp->bif_dlt == DLT_PKTAP &&
2998 !(d->bd_flags & BPF_WANT_PKTAP)) {
2999 continue;
3000 }
3001 break;
3002 }
3003 }
3004 if (bp != NULL) {
3005 opromisc = d->bd_promisc;
3006 if (bpf_detachd(d, proc) != 0) {
3007 return ENXIO;
3008 }
3009 error = bpf_attachd(d, bp);
3010 if (error != 0) {
3011 os_log_error(OS_LOG_DEFAULT,
3012 "bpf_setdlt: bpf%d bpf_attachd %s error %d",
3013 d->bd_dev_minor, if_name(bp->bif_ifp),
3014 error);
3015 return error;
3016 }
3017 reset_d(d);
3018 if (opromisc) {
3019 lck_mtx_unlock(bpf_mlock);
3020 error = ifnet_set_promiscuous(bp->bif_ifp, 1);
3021 lck_mtx_lock(bpf_mlock);
3022 if (error != 0) {
3023 os_log_error(OS_LOG_DEFAULT,
3024 "bpf_setdlt: bpf%d ifpromisc %s error %d",
3025 d->bd_dev_minor, if_name(bp->bif_ifp), error);
3026 } else {
3027 d->bd_promisc = 1;
3028 }
3029 }
3030 }
3031 return bp == NULL ? EINVAL : 0;
3032 }
3033
3034 static int
bpf_set_traffic_class(struct bpf_d * d,int tc)3035 bpf_set_traffic_class(struct bpf_d *d, int tc)
3036 {
3037 int error = 0;
3038
3039 if (!SO_VALID_TC(tc)) {
3040 error = EINVAL;
3041 } else {
3042 d->bd_traffic_class = tc;
3043 }
3044
3045 return error;
3046 }
3047
3048 static void
bpf_set_packet_service_class(struct mbuf * m,int tc)3049 bpf_set_packet_service_class(struct mbuf *m, int tc)
3050 {
3051 if (!(m->m_flags & M_PKTHDR)) {
3052 return;
3053 }
3054
3055 VERIFY(SO_VALID_TC(tc));
3056 (void) m_set_service_class(m, so_tc2msc(tc));
3057 }
3058
3059 /*
3060 * Support for select()
3061 *
3062 * Return true iff the specific operation will not block indefinitely.
3063 * Otherwise, return false but make a note that a selwakeup() must be done.
3064 */
3065 int
bpfselect(dev_t dev,int which,void * wql,struct proc * proc)3066 bpfselect(dev_t dev, int which, void * wql, struct proc *proc)
3067 {
3068 struct bpf_d *d;
3069 int ret = 0;
3070
3071 lck_mtx_lock(bpf_mlock);
3072
3073 d = bpf_dtab[minor(dev)];
3074 if (d == NULL || d == BPF_DEV_RESERVED ||
3075 (d->bd_flags & BPF_CLOSING) != 0) {
3076 lck_mtx_unlock(bpf_mlock);
3077 return ENXIO;
3078 }
3079
3080 bpf_acquire_d(d);
3081
3082 if (d->bd_bif == NULL) {
3083 bpf_release_d(d);
3084 lck_mtx_unlock(bpf_mlock);
3085 return ENXIO;
3086 }
3087
3088 while (d->bd_hbuf_read) {
3089 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfselect", NULL);
3090 }
3091
3092 if ((d->bd_flags & BPF_CLOSING) != 0) {
3093 bpf_release_d(d);
3094 lck_mtx_unlock(bpf_mlock);
3095 return ENXIO;
3096 }
3097
3098 switch (which) {
3099 case FREAD:
3100 if (d->bd_hlen != 0 ||
3101 ((d->bd_immediate ||
3102 d->bd_state == BPF_TIMED_OUT) && d->bd_slen != 0)) {
3103 ret = 1; /* read has data to return */
3104 } else {
3105 /*
3106 * Read has no data to return.
3107 * Make the select wait, and start a timer if
3108 * necessary.
3109 */
3110 selrecord(proc, &d->bd_sel, wql);
3111 bpf_start_timer(d);
3112 }
3113 break;
3114
3115 case FWRITE:
3116 /* can't determine whether a write would block */
3117 ret = 1;
3118 break;
3119 }
3120
3121 bpf_release_d(d);
3122 lck_mtx_unlock(bpf_mlock);
3123
3124 return ret;
3125 }
3126
3127 /*
3128 * Support for kevent() system call. Register EVFILT_READ filters and
3129 * reject all others.
3130 */
3131 int bpfkqfilter(dev_t dev, struct knote *kn);
3132 static void filt_bpfdetach(struct knote *);
3133 static int filt_bpfread(struct knote *, long);
3134 static int filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev);
3135 static int filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev);
3136
3137 SECURITY_READ_ONLY_EARLY(struct filterops) bpfread_filtops = {
3138 .f_isfd = 1,
3139 .f_detach = filt_bpfdetach,
3140 .f_event = filt_bpfread,
3141 .f_touch = filt_bpftouch,
3142 .f_process = filt_bpfprocess,
3143 };
3144
3145 static int
filt_bpfread_common(struct knote * kn,struct kevent_qos_s * kev,struct bpf_d * d)3146 filt_bpfread_common(struct knote *kn, struct kevent_qos_s *kev, struct bpf_d *d)
3147 {
3148 int ready = 0;
3149 int64_t data = 0;
3150
3151 if (d->bd_immediate) {
3152 /*
3153 * If there's data in the hold buffer, it's the
3154 * amount of data a read will return.
3155 *
3156 * If there's no data in the hold buffer, but
3157 * there's data in the store buffer, a read will
3158 * immediately rotate the store buffer to the
3159 * hold buffer, the amount of data in the store
3160 * buffer is the amount of data a read will
3161 * return.
3162 *
3163 * If there's no data in either buffer, we're not
3164 * ready to read.
3165 */
3166 data = (d->bd_hlen == 0 || d->bd_hbuf_read ?
3167 d->bd_slen : d->bd_hlen);
3168 int64_t lowwat = knote_low_watermark(kn);
3169 if (lowwat > d->bd_bufsize) {
3170 lowwat = d->bd_bufsize;
3171 }
3172 ready = (data >= lowwat);
3173 } else {
3174 /*
3175 * If there's data in the hold buffer, it's the
3176 * amount of data a read will return.
3177 *
3178 * If there's no data in the hold buffer, but
3179 * there's data in the store buffer, if the
3180 * timer has expired a read will immediately
3181 * rotate the store buffer to the hold buffer,
3182 * so the amount of data in the store buffer is
3183 * the amount of data a read will return.
3184 *
3185 * If there's no data in either buffer, or there's
3186 * no data in the hold buffer and the timer hasn't
3187 * expired, we're not ready to read.
3188 */
3189 data = ((d->bd_hlen == 0 || d->bd_hbuf_read) &&
3190 d->bd_state == BPF_TIMED_OUT ? d->bd_slen : d->bd_hlen);
3191 ready = (data > 0);
3192 }
3193 if (!ready) {
3194 bpf_start_timer(d);
3195 } else if (kev) {
3196 knote_fill_kevent(kn, kev, data);
3197 }
3198
3199 return ready;
3200 }
3201
3202 int
bpfkqfilter(dev_t dev,struct knote * kn)3203 bpfkqfilter(dev_t dev, struct knote *kn)
3204 {
3205 struct bpf_d *d;
3206 int res;
3207
3208 /*
3209 * Is this device a bpf?
3210 */
3211 if (major(dev) != CDEV_MAJOR || kn->kn_filter != EVFILT_READ) {
3212 knote_set_error(kn, EINVAL);
3213 return 0;
3214 }
3215
3216 lck_mtx_lock(bpf_mlock);
3217
3218 d = bpf_dtab[minor(dev)];
3219
3220 if (d == NULL || d == BPF_DEV_RESERVED ||
3221 (d->bd_flags & BPF_CLOSING) != 0 ||
3222 d->bd_bif == NULL) {
3223 lck_mtx_unlock(bpf_mlock);
3224 knote_set_error(kn, ENXIO);
3225 return 0;
3226 }
3227
3228 kn->kn_filtid = EVFILTID_BPFREAD;
3229 knote_kn_hook_set_raw(kn, d);
3230 KNOTE_ATTACH(&d->bd_sel.si_note, kn);
3231 d->bd_flags |= BPF_KNOTE;
3232
3233 /* capture the current state */
3234 res = filt_bpfread_common(kn, NULL, d);
3235
3236 lck_mtx_unlock(bpf_mlock);
3237
3238 return res;
3239 }
3240
3241 static void
filt_bpfdetach(struct knote * kn)3242 filt_bpfdetach(struct knote *kn)
3243 {
3244 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3245
3246 lck_mtx_lock(bpf_mlock);
3247 if (d->bd_flags & BPF_KNOTE) {
3248 KNOTE_DETACH(&d->bd_sel.si_note, kn);
3249 d->bd_flags &= ~BPF_KNOTE;
3250 }
3251 lck_mtx_unlock(bpf_mlock);
3252 }
3253
3254 static int
filt_bpfread(struct knote * kn,long hint)3255 filt_bpfread(struct knote *kn, long hint)
3256 {
3257 #pragma unused(hint)
3258 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3259
3260 return filt_bpfread_common(kn, NULL, d);
3261 }
3262
3263 static int
filt_bpftouch(struct knote * kn,struct kevent_qos_s * kev)3264 filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev)
3265 {
3266 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3267 int res;
3268
3269 lck_mtx_lock(bpf_mlock);
3270
3271 /* save off the lowat threshold and flag */
3272 kn->kn_sdata = kev->data;
3273 kn->kn_sfflags = kev->fflags;
3274
3275 /* output data will be re-generated here */
3276 res = filt_bpfread_common(kn, NULL, d);
3277
3278 lck_mtx_unlock(bpf_mlock);
3279
3280 return res;
3281 }
3282
3283 static int
filt_bpfprocess(struct knote * kn,struct kevent_qos_s * kev)3284 filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev)
3285 {
3286 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3287 int res;
3288
3289 lck_mtx_lock(bpf_mlock);
3290 res = filt_bpfread_common(kn, kev, d);
3291 lck_mtx_unlock(bpf_mlock);
3292
3293 return res;
3294 }
3295
3296 /*
3297 * Copy data from an mbuf chain into a buffer. This code is derived
3298 * from m_copydata in kern/uipc_mbuf.c.
3299 */
3300 static void
bpf_mcopy(struct mbuf * m,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3301 bpf_mcopy(struct mbuf *m, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3302 {
3303 u_int count;
3304
3305 while (offset >= m->m_len) {
3306 offset -= m->m_len;
3307 m = m->m_next;
3308 if (m == NULL) {
3309 panic("bpf_mcopy");
3310 }
3311 continue;
3312 }
3313
3314 while (len > 0) {
3315 if (m == NULL) {
3316 panic("bpf_mcopy");
3317 }
3318 count = MIN(m->m_len - (u_int)offset, (u_int)len);
3319 bcopy(m_mtod_current(m) + offset, dst, count);
3320 m = m->m_next;
3321 dst += count;
3322 len -= count;
3323 offset = 0;
3324 }
3325 }
3326
3327 static inline void
bpf_tap_imp(ifnet_t ifp,u_int32_t dlt,struct bpf_packet * bpf_pkt,int outbound)3328 bpf_tap_imp(
3329 ifnet_t ifp,
3330 u_int32_t dlt,
3331 struct bpf_packet *bpf_pkt,
3332 int outbound)
3333 {
3334 struct bpf_d *d;
3335 u_int slen;
3336 struct bpf_if *bp;
3337
3338 /*
3339 * It's possible that we get here after the bpf descriptor has been
3340 * detached from the interface; in such a case we simply return.
3341 * Lock ordering is important since we can be called asynchronously
3342 * (from IOKit) to process an inbound packet; when that happens
3343 * we would have been holding its "gateLock" and will be acquiring
3344 * "bpf_mlock" upon entering this routine. Due to that, we release
3345 * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will
3346 * acquire "gateLock" in the IOKit), in order to avoid a deadlock
3347 * when a ifnet_set_promiscuous request simultaneously collides with
3348 * an inbound packet being passed into the tap callback.
3349 */
3350 lck_mtx_lock(bpf_mlock);
3351 if (ifp->if_bpf == NULL) {
3352 lck_mtx_unlock(bpf_mlock);
3353 return;
3354 }
3355 for (bp = ifp->if_bpf; bp != NULL; bp = bp->bif_next) {
3356 if (bp->bif_ifp != ifp) {
3357 /* wrong interface */
3358 bp = NULL;
3359 break;
3360 }
3361 if (dlt == 0 || bp->bif_dlt == dlt) {
3362 /* tapping default DLT or DLT matches */
3363 break;
3364 }
3365 }
3366 if (bp == NULL) {
3367 goto done;
3368 }
3369 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) {
3370 struct bpf_packet *bpf_pkt_saved = bpf_pkt;
3371 u_char *bpf_pkt_ptr = (u_char *)(struct bpf_packet *__bidi_indexable)bpf_pkt;
3372 struct bpf_packet bpf_pkt_tmp = {};
3373 struct pktap_header_buffer bpfp_header_tmp = {};
3374
3375 if (outbound && (d->bd_direction & BPF_D_OUT) == 0) {
3376 continue;
3377 }
3378 if (!outbound && (d->bd_direction & BPF_D_IN) == 0) {
3379 continue;
3380 }
3381
3382 ++d->bd_rcount;
3383 slen = bpf_filter(d->bd_filter, d->bd_filter_len,
3384 bpf_pkt_ptr,
3385 (u_int)bpf_pkt->bpfp_total_length, 0);
3386
3387 if (slen != 0) {
3388 if (bp->bif_ifp->if_type == IFT_PKTAP &&
3389 bp->bif_dlt == DLT_PKTAP) {
3390 if (d->bd_flags & BPF_TRUNCATE) {
3391 slen = min(slen, get_pkt_trunc_len(bpf_pkt));
3392 }
3393 /*
3394 * Need to copy the bpf_pkt because the conversion
3395 * to v2 pktap header modifies the content of the
3396 * bpfp_header
3397 */
3398 if ((d->bd_flags & BPF_PKTHDRV2) &&
3399 bpf_pkt->bpfp_header_length <= sizeof(bpfp_header_tmp)) {
3400 bpf_pkt_tmp = *bpf_pkt;
3401 bpf_pkt = &bpf_pkt_tmp;
3402
3403 memcpy(&bpfp_header_tmp, bpf_pkt->bpfp_header,
3404 bpf_pkt->bpfp_header_length);
3405
3406 bpf_pkt->bpfp_header = &bpfp_header_tmp;
3407 bpf_pkt->bpfp_header_length = bpf_pkt->bpfp_header_length;
3408
3409 convert_to_pktap_header_to_v2(bpf_pkt,
3410 !!(d->bd_flags & BPF_TRUNCATE));
3411 }
3412 }
3413 ++d->bd_fcount;
3414 catchpacket(d, bpf_pkt, slen, outbound);
3415 }
3416 bpf_pkt = bpf_pkt_saved;
3417 }
3418
3419 done:
3420 lck_mtx_unlock(bpf_mlock);
3421 }
3422
3423 static inline void
bpf_tap_mbuf(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen,int outbound)3424 bpf_tap_mbuf(
3425 ifnet_t ifp,
3426 u_int32_t dlt,
3427 mbuf_t m,
3428 void *__sized_by(hlen) hdr,
3429 size_t hlen,
3430 int outbound)
3431 {
3432 struct bpf_packet bpf_pkt;
3433 mbuf_ref_t m0;
3434
3435 if (ifp->if_bpf == NULL) {
3436 /* quickly check without taking lock */
3437 return;
3438 }
3439 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3440 bpf_pkt.bpfp_mbuf = m;
3441 bpf_pkt.bpfp_total_length = 0;
3442 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
3443 bpf_pkt.bpfp_total_length += m0->m_len;
3444 }
3445 if (hdr != NULL) {
3446 bpf_pkt.bpfp_header = hdr;
3447 bpf_pkt.bpfp_header_length = hlen;
3448 bpf_pkt.bpfp_total_length += hlen;
3449 } else {
3450 bpf_pkt.bpfp_header = NULL;
3451 bpf_pkt.bpfp_header_length = 0;
3452 }
3453 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3454 }
3455
3456 void
bpf_tap_out(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen)3457 bpf_tap_out(
3458 ifnet_t ifp,
3459 u_int32_t dlt,
3460 mbuf_t m,
3461 void *__sized_by(hlen) hdr,
3462 size_t hlen)
3463 {
3464 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 1);
3465 }
3466
3467 void
bpf_tap_in(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen)3468 bpf_tap_in(
3469 ifnet_t ifp,
3470 u_int32_t dlt,
3471 mbuf_t m,
3472 void *__sized_by(hlen) hdr,
3473 size_t hlen)
3474 {
3475 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 0);
3476 }
3477
3478 /* Callback registered with Ethernet driver. */
3479 static int
bpf_tap_callback(struct ifnet * ifp,struct mbuf * m)3480 bpf_tap_callback(struct ifnet *ifp, struct mbuf *m)
3481 {
3482 bpf_tap_mbuf(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL);
3483
3484 return 0;
3485 }
3486
3487 #if SKYWALK
3488 #include <skywalk/os_skywalk_private.h>
3489
3490 static void
bpf_pktcopy(kern_packet_t pkt,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3491 bpf_pktcopy(kern_packet_t pkt, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3492 {
3493 kern_buflet_t buflet = NULL;
3494 size_t count;
3495
3496 while (len > 0) {
3497 uint8_t *addr;
3498 uint32_t buflet_length;
3499 uint32_t buflet_offset;
3500 uint32_t limit;
3501
3502 buflet = kern_packet_get_next_buflet(pkt, buflet);
3503 VERIFY(buflet != NULL);
3504 limit = kern_buflet_get_data_limit(buflet);
3505 buflet_offset = kern_buflet_get_data_offset(buflet);
3506 addr = __unsafe_forge_bidi_indexable(uint8_t *,
3507 (uintptr_t)kern_buflet_get_data_address(buflet) + buflet_offset,
3508 limit - buflet_offset);
3509 VERIFY(addr != NULL);
3510 buflet_length = kern_buflet_get_data_length(buflet);
3511 if (offset >= buflet_length) {
3512 offset -= buflet_length;
3513 continue;
3514 }
3515 count = MIN(buflet_length - offset, len);
3516 bcopy((void *)(addr + offset), (void *)dst, count);
3517 dst += count;
3518 len -= count;
3519 offset = 0;
3520 }
3521 }
3522
3523 static inline void
bpf_tap_packet(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen,int outbound)3524 bpf_tap_packet(
3525 ifnet_t ifp,
3526 u_int32_t dlt,
3527 kern_packet_t pkt,
3528 void *__sized_by(hlen) hdr,
3529 size_t hlen,
3530 int outbound)
3531 {
3532 struct bpf_packet bpf_pkt;
3533 struct mbuf * m;
3534
3535 if (ifp->if_bpf == NULL) {
3536 /* quickly check without taking lock */
3537 return;
3538 }
3539 m = kern_packet_get_mbuf(pkt);
3540 if (m != NULL) {
3541 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3542 bpf_pkt.bpfp_mbuf = m;
3543 bpf_pkt.bpfp_total_length = m_length(m);
3544 } else {
3545 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_PKT;
3546 bpf_pkt.bpfp_pkt = pkt;
3547 if (PKT_ADDR(pkt)->pkt_pflags & PKT_F_TRUNCATED) {
3548 struct __kern_buflet *bft = kern_packet_get_next_buflet(pkt, NULL);
3549 bpf_pkt.bpfp_total_length = kern_buflet_get_data_length(bft);
3550 } else {
3551 bpf_pkt.bpfp_total_length = kern_packet_get_data_length(pkt);
3552 }
3553 }
3554 bpf_pkt.bpfp_header = hdr;
3555 bpf_pkt.bpfp_header_length = hlen;
3556 if (hlen != 0) {
3557 bpf_pkt.bpfp_total_length += hlen;
3558 }
3559 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3560 }
3561
3562 void
bpf_tap_packet_out(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen)3563 bpf_tap_packet_out(
3564 ifnet_t ifp,
3565 u_int32_t dlt,
3566 kern_packet_t pkt,
3567 void *__sized_by(hlen) hdr,
3568 size_t hlen)
3569 {
3570 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 1);
3571 }
3572
3573 void
bpf_tap_packet_in(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen)3574 bpf_tap_packet_in(
3575 ifnet_t ifp,
3576 u_int32_t dlt,
3577 kern_packet_t pkt,
3578 void *__sized_by(hlen) hdr,
3579 size_t hlen)
3580 {
3581 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 0);
3582 }
3583
3584 #endif /* SKYWALK */
3585
3586 static errno_t
bpf_copydata(struct bpf_packet * pkt,size_t off,size_t len,void * __sized_by (len)out_data)3587 bpf_copydata(struct bpf_packet *pkt, size_t off, size_t len, void *__sized_by(len) out_data)
3588 {
3589 errno_t err = 0;
3590 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
3591 err = mbuf_copydata(pkt->bpfp_mbuf, off, len, out_data);
3592 #if SKYWALK
3593 } else if (pkt->bpfp_type == BPF_PACKET_TYPE_PKT) {
3594 err = kern_packet_copy_bytes(pkt->bpfp_pkt, off, len, out_data);
3595 #endif /* SKYWALK */
3596 } else {
3597 err = EINVAL;
3598 }
3599
3600 return err;
3601 }
3602
3603 static void
copy_bpf_packet_offset(struct bpf_packet * pkt,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3604 copy_bpf_packet_offset(struct bpf_packet * pkt, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3605 {
3606 /* copy the optional header */
3607 if (offset < pkt->bpfp_header_length) {
3608 size_t count = MIN(len, pkt->bpfp_header_length - offset);
3609 caddr_t src = pkt->bpfp_header;
3610 bcopy(src + offset, dst, count);
3611 dst += count;
3612 len -= count;
3613 offset = 0;
3614 } else {
3615 offset -= pkt->bpfp_header_length;
3616 }
3617
3618 if (len == 0) {
3619 /* nothing past the header */
3620 return;
3621 }
3622 /* copy the packet */
3623 switch (pkt->bpfp_type) {
3624 case BPF_PACKET_TYPE_MBUF:
3625 bpf_mcopy(pkt->bpfp_mbuf, dst, len, offset);
3626 break;
3627 #if SKYWALK
3628 case BPF_PACKET_TYPE_PKT:
3629 bpf_pktcopy(pkt->bpfp_pkt, dst, len, offset);
3630 break;
3631 #endif /* SKYWALK */
3632 default:
3633 break;
3634 }
3635 }
3636
3637 static void
copy_bpf_packet(struct bpf_packet * pkt,uint8_t * __sized_by (len)dst,size_t len)3638 copy_bpf_packet(struct bpf_packet * pkt, uint8_t *__sized_by(len) dst, size_t len)
3639 {
3640 copy_bpf_packet_offset(pkt, dst, len, 0);
3641 }
3642
3643 static uint32_t
get_esp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3644 get_esp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3645 const uint32_t remaining_caplen)
3646 {
3647 /*
3648 * For some reason tcpdump expects to have one byte beyond the ESP header
3649 */
3650 uint32_t trunc_len = ESP_HDR_SIZE + 1;
3651
3652 if (trunc_len > remaining_caplen) {
3653 return remaining_caplen;
3654 }
3655
3656 return trunc_len;
3657 }
3658
3659 static uint32_t
get_isakmp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3660 get_isakmp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3661 const uint32_t remaining_caplen)
3662 {
3663 /*
3664 * Include the payload generic header
3665 */
3666 uint32_t trunc_len = ISAKMP_HDR_SIZE;
3667
3668 if (trunc_len > remaining_caplen) {
3669 return remaining_caplen;
3670 }
3671
3672 return trunc_len;
3673 }
3674
3675 static uint32_t
get_isakmp_natt_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3676 get_isakmp_natt_trunc_len(struct bpf_packet *pkt, uint32_t off,
3677 const uint32_t remaining_caplen)
3678 {
3679 int err = 0;
3680 uint32_t trunc_len = 0;
3681 char payload[remaining_caplen];
3682
3683 err = bpf_copydata(pkt, off, remaining_caplen, payload);
3684 if (err != 0) {
3685 return remaining_caplen;
3686 }
3687 /*
3688 * They are three cases:
3689 * - IKE: payload start with 4 bytes header set to zero before ISAKMP header
3690 * - keep alive: 1 byte payload
3691 * - otherwise it's ESP
3692 */
3693 if (remaining_caplen >= 4 &&
3694 payload[0] == 0 && payload[1] == 0 &&
3695 payload[2] == 0 && payload[3] == 0) {
3696 trunc_len = 4 + get_isakmp_trunc_len(pkt, off + 4, remaining_caplen - 4);
3697 } else if (remaining_caplen == 1) {
3698 trunc_len = 1;
3699 } else {
3700 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3701 }
3702
3703 if (trunc_len > remaining_caplen) {
3704 return remaining_caplen;
3705 }
3706
3707 return trunc_len;
3708 }
3709
3710 static uint32_t
get_udp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3711 get_udp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3712 {
3713 int err = 0;
3714 uint32_t trunc_len = sizeof(struct udphdr); /* By default no UDP payload */
3715
3716 if (trunc_len >= remaining_caplen) {
3717 return remaining_caplen;
3718 }
3719
3720 struct udphdr udphdr;
3721 err = bpf_copydata(pkt, off, sizeof(struct udphdr), &udphdr);
3722 if (err != 0) {
3723 return remaining_caplen;
3724 }
3725
3726 u_short sport, dport;
3727
3728 sport = EXTRACT_SHORT(&udphdr.uh_sport);
3729 dport = EXTRACT_SHORT(&udphdr.uh_dport);
3730
3731 if (dport == PORT_DNS || sport == PORT_DNS) {
3732 /*
3733 * Full UDP payload for DNS
3734 */
3735 trunc_len = remaining_caplen;
3736 } else if ((sport == PORT_BOOTPS && dport == PORT_BOOTPC) ||
3737 (sport == PORT_BOOTPC && dport == PORT_BOOTPS)) {
3738 /*
3739 * Full UDP payload for BOOTP and DHCP
3740 */
3741 trunc_len = remaining_caplen;
3742 } else if (dport == PORT_ISAKMP && sport == PORT_ISAKMP) {
3743 /*
3744 * Return the ISAKMP header
3745 */
3746 trunc_len += get_isakmp_trunc_len(pkt, off + sizeof(struct udphdr),
3747 remaining_caplen - sizeof(struct udphdr));
3748 } else if (dport == PORT_ISAKMP_NATT && sport == PORT_ISAKMP_NATT) {
3749 trunc_len += get_isakmp_natt_trunc_len(pkt, off + sizeof(struct udphdr),
3750 remaining_caplen - sizeof(struct udphdr));
3751 }
3752 if (trunc_len >= remaining_caplen) {
3753 return remaining_caplen;
3754 }
3755
3756 return trunc_len;
3757 }
3758
3759 static uint32_t
get_tcp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3760 get_tcp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3761 {
3762 int err = 0;
3763 uint32_t trunc_len = sizeof(struct tcphdr); /* By default no TCP payload */
3764 if (trunc_len >= remaining_caplen) {
3765 return remaining_caplen;
3766 }
3767
3768 struct tcphdr tcphdr;
3769 err = bpf_copydata(pkt, off, sizeof(struct tcphdr), &tcphdr);
3770 if (err != 0) {
3771 return remaining_caplen;
3772 }
3773
3774 u_short sport, dport;
3775 sport = EXTRACT_SHORT(&tcphdr.th_sport);
3776 dport = EXTRACT_SHORT(&tcphdr.th_dport);
3777
3778 if (dport == PORT_DNS || sport == PORT_DNS) {
3779 /*
3780 * Full TCP payload for DNS
3781 */
3782 trunc_len = remaining_caplen;
3783 } else {
3784 trunc_len = (uint16_t)(tcphdr.th_off << 2);
3785 }
3786 if (trunc_len >= remaining_caplen) {
3787 return remaining_caplen;
3788 }
3789
3790 return trunc_len;
3791 }
3792
3793 static uint32_t
get_proto_trunc_len(uint8_t proto,struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3794 get_proto_trunc_len(uint8_t proto, struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3795 {
3796 uint32_t trunc_len;
3797
3798 switch (proto) {
3799 case IPPROTO_ICMP: {
3800 /*
3801 * Full IMCP payload
3802 */
3803 trunc_len = remaining_caplen;
3804 break;
3805 }
3806 case IPPROTO_ICMPV6: {
3807 /*
3808 * Full IMCPV6 payload
3809 */
3810 trunc_len = remaining_caplen;
3811 break;
3812 }
3813 case IPPROTO_IGMP: {
3814 /*
3815 * Full IGMP payload
3816 */
3817 trunc_len = remaining_caplen;
3818 break;
3819 }
3820 case IPPROTO_UDP: {
3821 trunc_len = get_udp_trunc_len(pkt, off, remaining_caplen);
3822 break;
3823 }
3824 case IPPROTO_TCP: {
3825 trunc_len = get_tcp_trunc_len(pkt, off, remaining_caplen);
3826 break;
3827 }
3828 case IPPROTO_ESP: {
3829 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3830 break;
3831 }
3832 default: {
3833 /*
3834 * By default we only include the IP header
3835 */
3836 trunc_len = 0;
3837 break;
3838 }
3839 }
3840 if (trunc_len >= remaining_caplen) {
3841 return remaining_caplen;
3842 }
3843
3844 return trunc_len;
3845 }
3846
3847 static uint32_t
get_ip_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3848 get_ip_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3849 {
3850 int err = 0;
3851 uint32_t iplen = sizeof(struct ip);
3852 if (iplen >= remaining_caplen) {
3853 return remaining_caplen;
3854 }
3855
3856 struct ip iphdr;
3857 err = bpf_copydata(pkt, off, sizeof(struct ip), &iphdr);
3858 if (err != 0) {
3859 return remaining_caplen;
3860 }
3861
3862 uint8_t proto = 0;
3863
3864 iplen = (uint16_t)(iphdr.ip_hl << 2);
3865 if (iplen >= remaining_caplen) {
3866 return remaining_caplen;
3867 }
3868
3869 proto = iphdr.ip_p;
3870 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3871
3872 if (iplen >= remaining_caplen) {
3873 return remaining_caplen;
3874 }
3875
3876 return iplen;
3877 }
3878
3879 static uint32_t
get_ip6_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3880 get_ip6_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3881 {
3882 int err = 0;
3883 uint32_t iplen = sizeof(struct ip6_hdr);
3884 if (iplen >= remaining_caplen) {
3885 return remaining_caplen;
3886 }
3887
3888 struct ip6_hdr ip6hdr;
3889 err = bpf_copydata(pkt, off, sizeof(struct ip6_hdr), &ip6hdr);
3890 if (err != 0) {
3891 return remaining_caplen;
3892 }
3893
3894 uint8_t proto = 0;
3895
3896 /*
3897 * TBD: process the extension headers
3898 */
3899 proto = ip6hdr.ip6_nxt;
3900 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3901
3902 if (iplen >= remaining_caplen) {
3903 return remaining_caplen;
3904 }
3905
3906 return iplen;
3907 }
3908
3909 static uint32_t
get_ether_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3910 get_ether_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3911 {
3912 int err = 0;
3913 uint32_t ethlen = sizeof(struct ether_header);
3914 if (ethlen >= remaining_caplen) {
3915 return remaining_caplen;
3916 }
3917
3918 struct ether_header eh = {};
3919 err = bpf_copydata(pkt, off, sizeof(struct ether_header), &eh);
3920 if (err != 0) {
3921 return remaining_caplen;
3922 }
3923
3924 u_short type = EXTRACT_SHORT(&eh.ether_type);
3925 /* Include full ARP */
3926 if (type == ETHERTYPE_ARP) {
3927 ethlen = remaining_caplen;
3928 } else if (type == ETHERTYPE_IP) {
3929 ethlen += get_ip_trunc_len(pkt, off + sizeof(struct ether_header),
3930 remaining_caplen - ethlen);
3931 } else if (type == ETHERTYPE_IPV6) {
3932 ethlen += get_ip6_trunc_len(pkt, off + sizeof(struct ether_header),
3933 remaining_caplen - ethlen);
3934 } else {
3935 ethlen = MIN(BPF_MIN_PKT_SIZE, remaining_caplen);
3936 }
3937 return ethlen;
3938 }
3939
3940 static uint32_t
get_pkt_trunc_len(struct bpf_packet * pkt)3941 get_pkt_trunc_len(struct bpf_packet *pkt)
3942 {
3943 struct pktap_header *pktap = (struct pktap_header *) (pkt->bpfp_header);
3944 uint32_t in_pkt_len = 0;
3945 uint32_t out_pkt_len = 0;
3946 uint32_t tlen = 0;
3947 uint32_t pre_adjust; // L2 header not in mbuf or kern_packet
3948
3949 // bpfp_total_length must contain the BPF packet header
3950 assert3u(pkt->bpfp_total_length, >=, pkt->bpfp_header_length);
3951
3952 // The BPF packet header must contain the pktap header
3953 assert3u(pkt->bpfp_header_length, >=, pktap->pth_length);
3954
3955 // The pre frame length (L2 header) must be contained in the packet
3956 assert3u(pkt->bpfp_total_length, >=, pktap->pth_length + pktap->pth_frame_pre_length);
3957
3958 /*
3959 * pktap->pth_frame_pre_length is the L2 header length and accounts
3960 * for both L2 header in the packet payload and pre_adjust.
3961 *
3962 * pre_adjust represents an adjustment for a pseudo L2 header that is not
3963 * part of packet payload -- not in the mbuf or kern_packet -- and comes
3964 * just after the pktap header.
3965 *
3966 * pktap->pth_length is the size of the pktap header (exclude pre_adjust)
3967 *
3968 * pkt->bpfp_header_length is (pktap->pth_length + pre_adjust)
3969 */
3970 pre_adjust = (uint32_t)(pkt->bpfp_header_length - pktap->pth_length);
3971
3972 if (pktap->pth_iftype == IFT_ETHER) {
3973 /*
3974 * We need to parse the Ethernet header to find the network layer
3975 * protocol
3976 */
3977 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pre_adjust);
3978
3979 out_pkt_len = get_ether_trunc_len(pkt, 0, in_pkt_len);
3980
3981 tlen = pktap->pth_length + pre_adjust + out_pkt_len;
3982 } else {
3983 /*
3984 * For other interface types, we only know to parse IPv4 and IPv6.
3985 *
3986 * To get to the beginning of the IPv4 or IPv6 packet, we need to to skip
3987 * over the L2 header that is the actual packet payload (mbuf or kern_packet)
3988 */
3989 uint32_t off; // offset past the L2 header in the actual packet payload
3990
3991 off = pktap->pth_frame_pre_length - pre_adjust;
3992
3993 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pktap->pth_frame_pre_length);
3994
3995 if (pktap->pth_protocol_family == AF_INET) {
3996 out_pkt_len = get_ip_trunc_len(pkt, off, in_pkt_len);
3997 } else if (pktap->pth_protocol_family == AF_INET6) {
3998 out_pkt_len = get_ip6_trunc_len(pkt, off, in_pkt_len);
3999 } else {
4000 out_pkt_len = MIN(BPF_MIN_PKT_SIZE, in_pkt_len);
4001 }
4002 tlen = pktap->pth_length + pktap->pth_frame_pre_length + out_pkt_len;
4003 }
4004
4005 // Verify we do not overflow the buffer
4006 if (__improbable(tlen > pkt->bpfp_total_length)) {
4007 bool do_panic = bpf_debug != 0 ? true : false;
4008
4009 #if DEBUG
4010 do_panic = true;
4011 #endif /* DEBUG */
4012 if (do_panic) {
4013 panic("%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
4014 __func__, __LINE__,
4015 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
4016 } else {
4017 os_log(OS_LOG_DEFAULT,
4018 "%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
4019 __func__, __LINE__,
4020 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
4021 }
4022 bpf_trunc_overflow += 1;
4023 tlen = (uint32_t)pkt->bpfp_total_length;
4024 }
4025
4026 return tlen;
4027 }
4028
4029 static uint8_t
get_common_prefix_size(const void * __bidi_indexable a,const void * __bidi_indexable b,uint8_t max_bytes)4030 get_common_prefix_size(const void *__bidi_indexable a, const void *__bidi_indexable b, uint8_t max_bytes)
4031 {
4032 uint8_t max_words = max_bytes >> 2;
4033 const uint32_t *x = (const uint32_t *)a;
4034 const uint32_t *y = (const uint32_t *)b;
4035 uint8_t i;
4036
4037 for (i = 0; i < max_words; i++) {
4038 if (x[i] != y[i]) {
4039 break;
4040 }
4041 }
4042 return (uint8_t)(i << 2);
4043 }
4044
4045 /*
4046 * Move the packet data from interface memory (pkt) into the
4047 * store buffer. Return 1 if it's time to wakeup a listener (buffer full),
4048 * otherwise 0.
4049 */
4050 static void
catchpacket(struct bpf_d * d,struct bpf_packet * pkt,u_int snaplen,int outbound)4051 catchpacket(struct bpf_d *d, struct bpf_packet * pkt,
4052 u_int snaplen, int outbound)
4053 {
4054 struct bpf_hdr *hp;
4055 struct bpf_hdr_ext *ehp;
4056 uint32_t totlen, curlen;
4057 uint32_t hdrlen, caplen;
4058 int do_wakeup = 0;
4059 u_char *payload;
4060 struct timeval tv = { .tv_sec = 0, .tv_usec = 0 };
4061
4062 hdrlen = (d->bd_flags & BPF_EXTENDED_HDR) ? d->bd_bif->bif_exthdrlen :
4063 (d->bd_flags & BPF_COMP_REQ) ? d->bd_bif->bif_comphdrlen:
4064 d->bd_bif->bif_hdrlen;
4065 /*
4066 * Figure out how many bytes to move. If the packet is
4067 * greater or equal to the snapshot length, transfer that
4068 * much. Otherwise, transfer the whole packet (unless
4069 * we hit the buffer size limit).
4070 */
4071 totlen = hdrlen + MIN(snaplen, (int)pkt->bpfp_total_length);
4072 if (totlen > d->bd_bufsize) {
4073 totlen = d->bd_bufsize;
4074 }
4075
4076 if (hdrlen > totlen) {
4077 return;
4078 }
4079
4080 /*
4081 * Round up the end of the previous packet to the next longword.
4082 */
4083 curlen = BPF_WORDALIGN(d->bd_slen);
4084 if (curlen + totlen > d->bd_bufsize) {
4085 /*
4086 * This packet will overflow the storage buffer.
4087 * Rotate the buffers if we can, then wakeup any
4088 * pending reads.
4089 *
4090 * We cannot rotate buffers if a read is in progress
4091 * so drop the packet
4092 */
4093 if (d->bd_hbuf_read) {
4094 ++d->bd_dcount;
4095 return;
4096 }
4097
4098 if (d->bd_fbuf == NULL) {
4099 if (d->bd_headdrop == 0) {
4100 /*
4101 * We haven't completed the previous read yet,
4102 * so drop the packet.
4103 */
4104 ++d->bd_dcount;
4105 return;
4106 }
4107 /*
4108 * Drop the hold buffer as it contains older packets
4109 */
4110 d->bd_dcount += d->bd_hcnt;
4111 d->bd_fbuf = d->bd_hbuf;
4112 ROTATE_BUFFERS(d);
4113 } else {
4114 ROTATE_BUFFERS(d);
4115 }
4116 do_wakeup = 1;
4117 curlen = 0;
4118 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
4119 /*
4120 * Immediate mode is set, or the read timeout has
4121 * already expired during a select call. A packet
4122 * arrived, so the reader should be woken up.
4123 */
4124 do_wakeup = 1;
4125 }
4126
4127 /*
4128 * Append the bpf header.
4129 */
4130 if (d->bd_tstamp != BPF_T_NONE) {
4131 microtime(&tv);
4132 }
4133 if (d->bd_flags & BPF_EXTENDED_HDR) {
4134 ehp = (struct bpf_hdr_ext *)(void *)(d->bd_sbuf + curlen);
4135 memset(ehp, 0, sizeof(*ehp));
4136 ehp->bh_tstamp.tv_sec = (int)tv.tv_sec;
4137 ehp->bh_tstamp.tv_usec = tv.tv_usec;
4138
4139 ehp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
4140 ehp->bh_hdrlen = (u_short)hdrlen;
4141 caplen = ehp->bh_caplen = totlen - hdrlen;
4142 payload = (u_char *)ehp + hdrlen;
4143
4144 if (outbound) {
4145 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT;
4146 } else {
4147 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN;
4148 }
4149
4150 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
4151 mbuf_ref_t m = pkt->bpfp_mbuf;
4152
4153 if (m->m_pkthdr.pkt_ext_flags & PKTF_EXT_ULPN) {
4154 ehp->bh_pktflags |= BPF_PKTFLAGS_ULPN;
4155 }
4156
4157 if (outbound) {
4158 /* only do lookups on non-raw INPCB */
4159 if ((m->m_pkthdr.pkt_flags & (PKTF_FLOW_ID |
4160 PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)) ==
4161 (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC) &&
4162 m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
4163 ehp->bh_flowid = m->m_pkthdr.pkt_flowid;
4164 if (m->m_pkthdr.pkt_proto == IPPROTO_TCP) {
4165 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_TCP;
4166 } else if (m->m_pkthdr.pkt_proto == IPPROTO_UDP) {
4167 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_UDP;
4168 }
4169 }
4170 ehp->bh_svc = so_svc2tc(m->m_pkthdr.pkt_svc);
4171 if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) {
4172 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
4173 }
4174 if (m->m_pkthdr.pkt_flags & PKTF_START_SEQ) {
4175 ehp->bh_pktflags |= BPF_PKTFLAGS_START_SEQ;
4176 }
4177 if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT) {
4178 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
4179 }
4180 if (m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA) {
4181 ehp->bh_unsent_bytes =
4182 m->m_pkthdr.bufstatus_if;
4183 ehp->bh_unsent_snd =
4184 m->m_pkthdr.bufstatus_sndbuf;
4185 }
4186 ehp->bh_comp_gencnt = m->m_pkthdr.comp_gencnt;
4187 } else {
4188 if (m->m_pkthdr.pkt_flags & PKTF_WAKE_PKT) {
4189 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
4190 }
4191 }
4192 #if SKYWALK
4193 } else {
4194 kern_packet_t kern_pkt = pkt->bpfp_pkt;
4195 packet_flowid_t flowid = 0;
4196
4197 if (kern_packet_get_ulpn_flag(kern_pkt)) {
4198 ehp->bh_pktflags |= BPF_PKTFLAGS_ULPN;
4199 }
4200
4201 if (outbound) {
4202 /*
4203 * Note: pp_init() asserts that kern_packet_svc_class_t is equivalent
4204 * to mbuf_svc_class_t
4205 */
4206 ehp->bh_svc = so_svc2tc((mbuf_svc_class_t)kern_packet_get_service_class(kern_pkt));
4207 if (kern_packet_get_transport_retransmit(kern_pkt)) {
4208 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
4209 }
4210 if (kern_packet_get_transport_last_packet(kern_pkt)) {
4211 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
4212 }
4213 kern_packet_get_compression_generation_count(kern_pkt, &ehp->bh_comp_gencnt);
4214 } else {
4215 if (kern_packet_get_wake_flag(kern_pkt)) {
4216 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
4217 }
4218 }
4219 ehp->bh_trace_tag = kern_packet_get_trace_tag(kern_pkt);
4220 if (kern_packet_get_flowid(kern_pkt, &flowid) == 0) {
4221 ehp->bh_flowid = flowid;
4222 }
4223 #endif /* SKYWALK */
4224 }
4225 } else {
4226 hp = (struct bpf_hdr *)(void *)(d->bd_sbuf + curlen);
4227 memset(hp, 0, BPF_WORDALIGN(sizeof(*hp)));
4228 hp->bh_tstamp.tv_sec = (int)tv.tv_sec;
4229 hp->bh_tstamp.tv_usec = tv.tv_usec;
4230 hp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
4231 hp->bh_hdrlen = (u_short)hdrlen;
4232 caplen = hp->bh_caplen = totlen - hdrlen;
4233 payload = (u_char *)hp + hdrlen;
4234 }
4235 if (d->bd_flags & BPF_COMP_REQ) {
4236 uint8_t common_prefix_size = 0;
4237 uint8_t copy_len = MIN((uint8_t)caplen, BPF_HDR_COMP_LEN_MAX);
4238
4239 copy_bpf_packet(pkt, (uint8_t *__bidi_indexable)d->bd_prev_fbuf, copy_len);
4240
4241 if (d->bd_prev_slen != 0) {
4242 common_prefix_size = get_common_prefix_size(d->bd_prev_fbuf,
4243 d->bd_prev_sbuf, MIN(copy_len, d->bd_prev_slen));
4244 }
4245
4246 if (d->bd_flags & BPF_COMP_ENABLED) {
4247 assert3u(caplen, >=, common_prefix_size);
4248 copy_bpf_packet_offset(pkt, payload, caplen - common_prefix_size,
4249 common_prefix_size);
4250 d->bd_slen = curlen + totlen - common_prefix_size;
4251 } else {
4252 copy_bpf_packet(pkt, payload, caplen);
4253 d->bd_slen = curlen + totlen;
4254 }
4255
4256 /*
4257 * Update the caplen only if compression is enabled -- the caller
4258 * must pay attention to bpf_hdr_comp_enable
4259 */
4260 if (d->bd_flags & BPF_EXTENDED_HDR) {
4261 ehp->bh_complen = common_prefix_size;
4262 if (d->bd_flags & BPF_COMP_ENABLED) {
4263 ehp->bh_caplen -= common_prefix_size;
4264 }
4265 } else {
4266 struct bpf_comp_hdr *hcp;
4267
4268 hcp = (struct bpf_comp_hdr *)(void *)(d->bd_sbuf + curlen);
4269 hcp->bh_complen = common_prefix_size;
4270 if (d->bd_flags & BPF_COMP_ENABLED) {
4271 hcp->bh_caplen -= common_prefix_size;
4272 }
4273 }
4274
4275 if (common_prefix_size > 0) {
4276 d->bd_bcs.bcs_total_compressed_prefix_size += common_prefix_size;
4277 if (common_prefix_size > d->bd_bcs.bcs_max_compressed_prefix_size) {
4278 d->bd_bcs.bcs_max_compressed_prefix_size = common_prefix_size;
4279 }
4280 d->bd_bcs.bcs_count_compressed_prefix += 1;
4281 } else {
4282 d->bd_bcs.bcs_count_no_common_prefix += 1;
4283 }
4284
4285 /* The current compression buffer becomes the previous one */
4286 caddr_t tmp = d->bd_prev_sbuf;
4287 d->bd_prev_sbuf = d->bd_prev_fbuf;
4288 d->bd_prev_slen = copy_len;
4289 d->bd_prev_fbuf = tmp;
4290 } else {
4291 /*
4292 * Copy the packet data into the store buffer and update its length.
4293 */
4294 copy_bpf_packet(pkt, payload, caplen);
4295 d->bd_slen = curlen + totlen;
4296 }
4297 d->bd_scnt += 1;
4298 d->bd_bcs.bcs_total_hdr_size += pkt->bpfp_header_length;
4299 d->bd_bcs.bcs_total_size += caplen;
4300
4301 if (do_wakeup) {
4302 bpf_wakeup(d);
4303 }
4304 }
4305
4306 static void
bpf_freebufs(struct bpf_d * d)4307 bpf_freebufs(struct bpf_d *d)
4308 {
4309 if (d->bd_sbuf != NULL) {
4310 kfree_data_addr(d->bd_sbuf);
4311 }
4312 if (d->bd_hbuf != NULL) {
4313 kfree_data_addr(d->bd_hbuf);
4314 }
4315 if (d->bd_fbuf != NULL) {
4316 kfree_data_addr(d->bd_fbuf);
4317 }
4318
4319 if (d->bd_prev_sbuf != NULL) {
4320 kfree_data_addr(d->bd_prev_sbuf);
4321 }
4322 if (d->bd_prev_fbuf != NULL) {
4323 kfree_data_addr(d->bd_prev_fbuf);
4324 }
4325 }
4326 /*
4327 * Initialize all nonzero fields of a descriptor.
4328 */
4329 static int
bpf_allocbufs(struct bpf_d * d)4330 bpf_allocbufs(struct bpf_d *d)
4331 {
4332 bpf_freebufs(d);
4333
4334 d->bd_fbuf = kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
4335 if (d->bd_fbuf == NULL) {
4336 goto nobufs;
4337 }
4338
4339 d->bd_sbuf = kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
4340 if (d->bd_sbuf == NULL) {
4341 goto nobufs;
4342 }
4343 d->bd_slen = 0;
4344 d->bd_hlen = 0;
4345 d->bd_scnt = 0;
4346 d->bd_hcnt = 0;
4347
4348 d->bd_prev_slen = 0;
4349 if (d->bd_flags & BPF_COMP_REQ) {
4350 d->bd_prev_sbuf = kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
4351 if (d->bd_prev_sbuf == NULL) {
4352 goto nobufs;
4353 }
4354 d->bd_prev_fbuf = kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
4355 if (d->bd_prev_fbuf == NULL) {
4356 goto nobufs;
4357 }
4358 }
4359 return 0;
4360 nobufs:
4361 bpf_freebufs(d);
4362 return ENOMEM;
4363 }
4364
4365 /*
4366 * Free buffers currently in use by a descriptor.
4367 * Called on close.
4368 */
4369 static void
bpf_freed(struct bpf_d * d)4370 bpf_freed(struct bpf_d *d)
4371 {
4372 /*
4373 * We don't need to lock out interrupts since this descriptor has
4374 * been detached from its interface and it yet hasn't been marked
4375 * free.
4376 */
4377 if (d->bd_hbuf_read || d->bd_hbuf_write) {
4378 panic("bpf buffer freed during read/write");
4379 }
4380
4381 bpf_freebufs(d);
4382
4383 if (d->bd_filter) {
4384 kfree_data_addr_sized_by(d->bd_filter, d->bd_filter_len);
4385 }
4386 }
4387
4388 /*
4389 * Attach an interface to bpf. driverp is a pointer to a (struct bpf_if *)
4390 * in the driver's softc; dlt is the link layer type; hdrlen is the fixed
4391 * size of the link header (variable length headers not yet supported).
4392 */
4393 void
bpfattach(struct ifnet * ifp,u_int dlt,u_int hdrlen)4394 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
4395 {
4396 bpf_attach(ifp, dlt, hdrlen, NULL, NULL);
4397 }
4398
4399 errno_t
bpf_attach(ifnet_t ifp,u_int32_t dlt,u_int32_t hdrlen,bpf_send_func send,bpf_tap_func tap)4400 bpf_attach(
4401 ifnet_t ifp,
4402 u_int32_t dlt,
4403 u_int32_t hdrlen,
4404 bpf_send_func send,
4405 bpf_tap_func tap)
4406 {
4407 struct bpf_if *bp;
4408 struct bpf_if *bp_new;
4409 struct bpf_if *bp_before_first = NULL;
4410 struct bpf_if *bp_first = NULL;
4411 struct bpf_if *bp_last = NULL;
4412 boolean_t found;
4413
4414 /*
4415 * Z_NOFAIL will cause a panic if the allocation fails
4416 */
4417 bp_new = kalloc_type(struct bpf_if, Z_WAITOK | Z_NOFAIL | Z_ZERO);
4418
4419 lck_mtx_lock(bpf_mlock);
4420
4421 /*
4422 * Check if this interface/dlt is already attached. Remember the
4423 * first and last attachment for this interface, as well as the
4424 * element before the first attachment.
4425 */
4426 found = FALSE;
4427 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
4428 if (bp->bif_ifp != ifp) {
4429 if (bp_first != NULL) {
4430 /* no more elements for this interface */
4431 break;
4432 }
4433 bp_before_first = bp;
4434 } else {
4435 if (bp->bif_dlt == dlt) {
4436 found = TRUE;
4437 break;
4438 }
4439 if (bp_first == NULL) {
4440 bp_first = bp;
4441 }
4442 bp_last = bp;
4443 }
4444 }
4445 if (found) {
4446 lck_mtx_unlock(bpf_mlock);
4447 os_log_error(OS_LOG_DEFAULT,
4448 "bpfattach - %s with dlt %d is already attached",
4449 if_name(ifp), dlt);
4450 kfree_type(struct bpf_if, bp_new);
4451 return EEXIST;
4452 }
4453
4454 bp_new->bif_ifp = ifp;
4455 bp_new->bif_dlt = dlt;
4456 bp_new->bif_send = send;
4457 bp_new->bif_tap = tap;
4458
4459 if (bp_first == NULL) {
4460 /* No other entries for this ifp */
4461 bp_new->bif_next = bpf_iflist;
4462 bpf_iflist = bp_new;
4463 } else {
4464 if (ifnet_type(ifp) == IFT_ETHER && dlt == DLT_EN10MB) {
4465 /* Make this the first entry for this interface */
4466 if (bp_before_first != NULL) {
4467 /* point the previous to us */
4468 bp_before_first->bif_next = bp_new;
4469 } else {
4470 /* we're the new head */
4471 bpf_iflist = bp_new;
4472 }
4473 bp_new->bif_next = bp_first;
4474 } else {
4475 /* Add this after the last entry for this interface */
4476 bp_new->bif_next = bp_last->bif_next;
4477 bp_last->bif_next = bp_new;
4478 }
4479 }
4480
4481 /*
4482 * Compute the length of the bpf header. This is not necessarily
4483 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
4484 * that the network layer header begins on a longword boundary (for
4485 * performance reasons and to alleviate alignment restrictions).
4486 */
4487 bp_new->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
4488 bp_new->bif_exthdrlen = BPF_WORDALIGN(hdrlen +
4489 sizeof(struct bpf_hdr_ext)) - hdrlen;
4490 bp_new->bif_comphdrlen = BPF_WORDALIGN(hdrlen +
4491 sizeof(struct bpf_comp_hdr)) - hdrlen;
4492
4493 /* Take a reference on the interface */
4494 ifnet_reference(ifp);
4495
4496 lck_mtx_unlock(bpf_mlock);
4497
4498 return 0;
4499 }
4500
4501 /*
4502 * Detach bpf from an interface. This involves detaching each descriptor
4503 * associated with the interface, and leaving bd_bif NULL. Notify each
4504 * descriptor as it's detached so that any sleepers wake up and get
4505 * ENXIO.
4506 */
4507 void
bpfdetach(struct ifnet * ifp)4508 bpfdetach(struct ifnet *ifp)
4509 {
4510 struct bpf_if *bp, *bp_prev, *bp_next;
4511 struct bpf_d *d;
4512
4513 if (bpf_debug != 0) {
4514 os_log(OS_LOG_DEFAULT, "%s: %s", __func__, if_name(ifp));
4515 }
4516
4517 lck_mtx_lock(bpf_mlock);
4518
4519 /*
4520 * Build the list of devices attached to that interface
4521 * that we need to free while keeping the lock to maintain
4522 * the integrity of the interface list
4523 */
4524 bp_prev = NULL;
4525 for (bp = bpf_iflist; bp != NULL; bp = bp_next) {
4526 bp_next = bp->bif_next;
4527
4528 if (ifp != bp->bif_ifp) {
4529 bp_prev = bp;
4530 continue;
4531 }
4532 /* Unlink from the interface list */
4533 if (bp_prev) {
4534 bp_prev->bif_next = bp->bif_next;
4535 } else {
4536 bpf_iflist = bp->bif_next;
4537 }
4538
4539 /* Detach the devices attached to the interface */
4540 while ((d = bp->bif_dlist) != NULL) {
4541 /*
4542 * Take an extra reference to prevent the device
4543 * from being freed when bpf_detachd() releases
4544 * the reference for the interface list
4545 */
4546 bpf_acquire_d(d);
4547
4548 /*
4549 * Wait for active read and writes to complete
4550 */
4551 while (d->bd_hbuf_read || d->bd_hbuf_write) {
4552 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfdetach", NULL);
4553 }
4554
4555 bpf_detachd(d, current_proc());
4556 bpf_wakeup(d);
4557 bpf_release_d(d);
4558 }
4559 ifnet_release(ifp);
4560 }
4561
4562 lck_mtx_unlock(bpf_mlock);
4563 }
4564
4565 void
bpf_init(__unused void * unused)4566 bpf_init(__unused void *unused)
4567 {
4568 int maj;
4569
4570 /* bpf_comp_hdr is an overlay of bpf_hdr */
4571 static_assert(BPF_WORDALIGN(sizeof(struct bpf_hdr)) == BPF_WORDALIGN(sizeof(struct bpf_comp_hdr)));
4572
4573 /* compression length must fits in a byte */
4574 static_assert(BPF_HDR_COMP_LEN_MAX <= UCHAR_MAX);
4575
4576 (void) PE_parse_boot_argn("bpf_hdr_comp", &bpf_hdr_comp_enable,
4577 sizeof(bpf_hdr_comp_enable));
4578
4579 if (bpf_devsw_installed == 0) {
4580 bpf_devsw_installed = 1;
4581 maj = cdevsw_add(CDEV_MAJOR, &bpf_cdevsw);
4582 if (maj == -1) {
4583 bpf_devsw_installed = 0;
4584 os_log_error(OS_LOG_DEFAULT,
4585 "bpf_init: failed to allocate a major number");
4586 return;
4587 }
4588
4589 for (int i = 0; i < NBPFILTER; i++) {
4590 bpf_make_dev_t(maj);
4591 }
4592 }
4593 }
4594
4595 static int
4596 sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS
4597 {
4598 #pragma unused(arg1, arg2)
4599 int i, err;
4600
4601 i = bpf_maxbufsize;
4602
4603 err = sysctl_handle_int(oidp, &i, 0, req);
4604 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4605 return err;
4606 }
4607
4608 if (i < 0 || i > BPF_BUFSIZE_CAP) {
4609 i = BPF_BUFSIZE_CAP;
4610 }
4611
4612 bpf_maxbufsize = i;
4613 return err;
4614 }
4615
4616 static int
4617 sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS
4618 {
4619 #pragma unused(arg1, arg2)
4620 int i, err;
4621
4622 i = BPF_BUFSIZE_CAP;
4623
4624 err = sysctl_handle_int(oidp, &i, 0, req);
4625 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4626 return err;
4627 }
4628
4629 return err;
4630 }
4631
4632 /*
4633 * Fill filter statistics
4634 */
4635 static void
bpfstats_fill_xbpf(struct xbpf_d * d,struct bpf_d * bd)4636 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
4637 {
4638 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
4639
4640 d->bd_structsize = sizeof(struct xbpf_d);
4641 d->bd_promisc = bd->bd_promisc != 0 ? 1 : 0;
4642 d->bd_immediate = d->bd_immediate != 0 ? 1 : 0;
4643 d->bd_hdrcmplt = bd->bd_hdrcmplt != 0 ? 1 : 0;
4644 d->bd_async = bd->bd_async != 0 ? 1 : 0;
4645 d->bd_headdrop = bd->bd_headdrop != 0 ? 1 : 0;
4646 d->bd_direction = (uint8_t)bd->bd_direction;
4647 d->bh_compreq = bd->bd_flags & BPF_COMP_REQ ? 1 : 0;
4648 d->bh_compenabled = bd->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
4649 d->bd_exthdr = bd->bd_flags & BPF_EXTENDED_HDR ? 1 : 0;
4650 d->bd_trunc = bd->bd_flags & BPF_TRUNCATE ? 1 : 0;
4651 d->bd_pkthdrv2 = bd->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
4652 d->bd_batch_write = bd->bd_flags & BPF_BATCH_WRITE ? 1 : 0;
4653 d->bd_divert_in = bd->bd_flags & BPF_DIVERT_IN ? 1 : 0;
4654
4655 d->bd_dev_minor = (uint8_t)bd->bd_dev_minor;
4656
4657 d->bd_sig = bd->bd_sig;
4658
4659 d->bd_rcount = bd->bd_rcount;
4660 d->bd_dcount = bd->bd_dcount;
4661 d->bd_fcount = bd->bd_fcount;
4662 d->bd_wcount = bd->bd_wcount;
4663 d->bd_wdcount = bd->bd_wdcount;
4664 d->bd_slen = bd->bd_slen;
4665 d->bd_hlen = bd->bd_hlen;
4666 d->bd_bufsize = bd->bd_bufsize;
4667 d->bd_pid = bd->bd_pid;
4668 if (bd->bd_bif != NULL && bd->bd_bif->bif_ifp != NULL) {
4669 strlcpy(d->bd_ifname,
4670 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
4671 }
4672
4673 d->bd_comp_count = bd->bd_bcs.bcs_count_compressed_prefix;
4674 d->bd_comp_size = bd->bd_bcs.bcs_total_compressed_prefix_size;
4675
4676 d->bd_scnt = bd->bd_scnt;
4677 d->bd_hcnt = bd->bd_hcnt;
4678
4679 d->bd_read_count = bd->bd_bcs.bcs_total_read;
4680 d->bd_fsize = bd->bd_bcs.bcs_total_size;
4681 }
4682
4683 /*
4684 * Handle `netstat -B' stats request
4685 */
4686 static int
4687 sysctl_bpf_stats SYSCTL_HANDLER_ARGS
4688 {
4689 int error;
4690 struct xbpf_d *xbdbuf;
4691 unsigned int x_cnt;
4692 vm_size_t buf_size;
4693
4694 if (req->oldptr == USER_ADDR_NULL) {
4695 return SYSCTL_OUT(req, 0, nbpfilter * sizeof(struct xbpf_d));
4696 }
4697 if (nbpfilter == 0) {
4698 return SYSCTL_OUT(req, 0, 0);
4699 }
4700 buf_size = req->oldlen;
4701 if (buf_size > BPF_MAX_DEVICES * sizeof(struct xbpf_d)) {
4702 buf_size = BPF_MAX_DEVICES * sizeof(struct xbpf_d);
4703 }
4704 xbdbuf = kalloc_data(buf_size, Z_WAITOK | Z_ZERO);
4705
4706 lck_mtx_lock(bpf_mlock);
4707 if (buf_size < (nbpfilter * sizeof(struct xbpf_d))) {
4708 lck_mtx_unlock(bpf_mlock);
4709 kfree_data(xbdbuf, buf_size);
4710 return ENOMEM;
4711 }
4712 x_cnt = 0;
4713 unsigned int i;
4714
4715 for (i = 0; i < nbpfilter; i++) {
4716 struct bpf_d *bd = bpf_dtab[i];
4717 struct xbpf_d *xbd;
4718
4719 if (bd == NULL || bd == BPF_DEV_RESERVED ||
4720 (bd->bd_flags & BPF_CLOSING) != 0) {
4721 continue;
4722 }
4723 VERIFY(x_cnt < nbpfilter);
4724
4725 xbd = &xbdbuf[x_cnt++];
4726 bpfstats_fill_xbpf(xbd, bd);
4727 }
4728 lck_mtx_unlock(bpf_mlock);
4729
4730 error = SYSCTL_OUT(req, xbdbuf, x_cnt * sizeof(struct xbpf_d));
4731 kfree_data(xbdbuf, buf_size);
4732 return error;
4733 }
4734