1 /*
2 * Copyright (c) 2000-2022 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1990, 1991, 1993
30 * The Regents of the University of California. All rights reserved.
31 *
32 * This code is derived from the Stanford/CMU enet packet filter,
33 * (net/enet.c) distributed as part of 4.3BSD, and code contributed
34 * to Berkeley by Steven McCanne and Van Jacobson both of Lawrence
35 * Berkeley Laboratory.
36 *
37 * Redistribution and use in source and binary forms, with or without
38 * modification, are permitted provided that the following conditions
39 * are met:
40 * 1. Redistributions of source code must retain the above copyright
41 * notice, this list of conditions and the following disclaimer.
42 * 2. Redistributions in binary form must reproduce the above copyright
43 * notice, this list of conditions and the following disclaimer in the
44 * documentation and/or other materials provided with the distribution.
45 * 3. All advertising materials mentioning features or use of this software
46 * must display the following acknowledgement:
47 * This product includes software developed by the University of
48 * California, Berkeley and its contributors.
49 * 4. Neither the name of the University nor the names of its contributors
50 * may be used to endorse or promote products derived from this software
51 * without specific prior written permission.
52 *
53 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
54 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
55 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
56 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
57 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
58 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
59 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
60 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
61 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
62 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
63 * SUCH DAMAGE.
64 *
65 * @(#)bpf.c 8.2 (Berkeley) 3/28/94
66 *
67 * $FreeBSD: src/sys/net/bpf.c,v 1.59.2.5 2001/01/05 04:49:09 jdp Exp $
68 */
69 /*
70 * NOTICE: This file was modified by SPARTA, Inc. in 2005 to introduce
71 * support for mandatory and extensible security protections. This notice
72 * is included in support of clause 2.2 (b) of the Apple Public License,
73 * Version 2.0.
74 */
75
76 #include "bpf.h"
77
78 #ifndef __GNUC__
79 #define inline
80 #else
81 #define inline __inline
82 #endif
83
84 #include <sys/param.h>
85 #include <sys/systm.h>
86 #include <sys/conf.h>
87 #include <sys/malloc.h>
88 #include <sys/mbuf.h>
89 #include <sys/time.h>
90 #include <sys/proc.h>
91 #include <sys/signalvar.h>
92 #include <sys/filio.h>
93 #include <sys/sockio.h>
94 #include <sys/ttycom.h>
95 #include <sys/filedesc.h>
96 #include <sys/uio_internal.h>
97 #include <sys/file_internal.h>
98 #include <sys/event.h>
99
100 #include <sys/poll.h>
101
102 #include <sys/socket.h>
103 #include <sys/socketvar.h>
104 #include <sys/vnode.h>
105
106 #include <net/if.h>
107 #include <net/bpf.h>
108 #include <net/bpfdesc.h>
109
110 #include <netinet/in.h>
111 #include <netinet/ip.h>
112 #include <netinet/ip6.h>
113 #include <netinet/in_pcb.h>
114 #include <netinet/in_var.h>
115 #include <netinet/ip_var.h>
116 #include <netinet/tcp.h>
117 #include <netinet/tcp_var.h>
118 #include <netinet/udp.h>
119 #include <netinet/udp_var.h>
120 #include <netinet/if_ether.h>
121 #include <netinet/isakmp.h>
122 #include <netinet6/esp.h>
123 #include <sys/kernel.h>
124 #include <sys/sysctl.h>
125 #include <net/firewire.h>
126
127 #include <miscfs/devfs/devfs.h>
128 #include <net/dlil.h>
129 #include <net/pktap.h>
130
131 #include <kern/assert.h>
132 #include <kern/locks.h>
133 #include <kern/thread_call.h>
134 #include <libkern/section_keywords.h>
135
136 #include <os/log.h>
137
138 #include <IOKit/IOBSD.h>
139
140 #include <net/sockaddr_utils.h>
141
142 extern int tvtohz(struct timeval *);
143 extern const char *proc_name_address(void *);
144
145 #define BPF_BUFSIZE 4096
146
147 #define PRINET 26 /* interruptible */
148
149 #define ISAKMP_HDR_SIZE (sizeof(struct isakmp) + sizeof(struct isakmp_gen))
150 #define ESP_HDR_SIZE sizeof(struct newesp)
151
152 #define BPF_WRITE_LEEWAY 18 /* space for link layer header */
153
154 #define BPF_WRITE_MAX 0x1000000 /* 16 MB arbitrary value */
155
156 typedef void (*pktcopyfunc_t)(const void *, void *, size_t);
157
158 /*
159 * The default read buffer size is patchable.
160 */
161 static unsigned int bpf_bufsize = BPF_BUFSIZE;
162 SYSCTL_INT(_debug, OID_AUTO, bpf_bufsize, CTLFLAG_RW | CTLFLAG_LOCKED,
163 &bpf_bufsize, 0, "");
164
165 __private_extern__ unsigned int bpf_maxbufsize = BPF_MAXBUFSIZE;
166 static int sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS;
167 SYSCTL_PROC(_debug, OID_AUTO, bpf_maxbufsize, CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
168 &bpf_maxbufsize, 0,
169 sysctl_bpf_maxbufsize, "I", "Default BPF max buffer size");
170
171 extern const int copysize_limit_panic;
172 #define BPF_BUFSIZE_CAP (copysize_limit_panic >> 1)
173 static int sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS;
174 SYSCTL_PROC(_debug, OID_AUTO, bpf_bufsize_cap, CTLTYPE_INT | CTLFLAG_RD | CTLFLAG_LOCKED,
175 0, 0,
176 sysctl_bpf_bufsize_cap, "I", "Upper limit on BPF max buffer size");
177
178 #define BPF_MAX_DEVICES 256
179 static unsigned int bpf_maxdevices = BPF_MAX_DEVICES;
180 SYSCTL_UINT(_debug, OID_AUTO, bpf_maxdevices, CTLFLAG_RD | CTLFLAG_LOCKED,
181 &bpf_maxdevices, 0, "");
182
183 /*
184 * bpf_wantpktap controls the defaul visibility of DLT_PKTAP
185 * For OS X is off by default so process need to use the ioctl BPF_WANT_PKTAP
186 * explicitly to be able to use DLT_PKTAP.
187 */
188 #if !XNU_TARGET_OS_OSX
189 static unsigned int bpf_wantpktap = 1;
190 #else /* XNU_TARGET_OS_OSX */
191 static unsigned int bpf_wantpktap = 0;
192 #endif /* XNU_TARGET_OS_OSX */
193 SYSCTL_UINT(_debug, OID_AUTO, bpf_wantpktap, CTLFLAG_RW | CTLFLAG_LOCKED,
194 &bpf_wantpktap, 0, "");
195
196 static int bpf_debug = 0;
197 SYSCTL_INT(_debug, OID_AUTO, bpf_debug, CTLFLAG_RW | CTLFLAG_LOCKED,
198 &bpf_debug, 0, "");
199
200 static unsigned long bpf_trunc_overflow = 0;
201 SYSCTL_ULONG(_debug, OID_AUTO, bpf_trunc_overflow, CTLFLAG_RD | CTLFLAG_LOCKED,
202 &bpf_trunc_overflow, "");
203
204 static int bpf_hdr_comp_enable = 1;
205 SYSCTL_INT(_debug, OID_AUTO, bpf_hdr_comp_enable, CTLFLAG_RW | CTLFLAG_LOCKED,
206 &bpf_hdr_comp_enable, 1, "");
207
208 static int sysctl_bpf_stats SYSCTL_HANDLER_ARGS;
209 SYSCTL_PROC(_debug, OID_AUTO, bpf_stats, CTLTYPE_STRUCT | CTLFLAG_RD | CTLFLAG_LOCKED,
210 0, 0,
211 sysctl_bpf_stats, "S", "BPF statistics");
212
213 /*
214 * bpf_iflist is the list of interfaces; each corresponds to an ifnet
215 * bpf_dtab holds pointer to the descriptors, indexed by minor device #
216 */
217 static struct bpf_if *bpf_iflist;
218 /*
219 * BSD now stores the bpf_d in the dev_t which is a struct
220 * on their system. Our dev_t is an int, so we still store
221 * the bpf_d in a separate table indexed by minor device #.
222 *
223 * The value stored in bpf_dtab[n] represent three states:
224 * NULL: device not opened
225 * BPF_DEV_RESERVED: device opening or closing
226 * other: device <n> opened with pointer to storage
227 */
228 static struct bpf_d *BPF_DEV_RESERVED = __unsafe_forge_single(struct bpf_d *, 1);
229 static unsigned int bpf_dtab_size = 0;
230 static struct bpf_d **__counted_by(bpf_dtab_size) bpf_dtab = NULL;
231 static unsigned int nbpfilter = 0;
232 static unsigned bpf_bpfd_cnt = 0;
233
234 static LCK_GRP_DECLARE(bpf_mlock_grp, "bpf");
235 static LCK_MTX_DECLARE(bpf_mlock_data, &bpf_mlock_grp);
236 static lck_mtx_t *const bpf_mlock = &bpf_mlock_data;
237
238 static int bpf_allocbufs(struct bpf_d *);
239 static errno_t bpf_attachd(struct bpf_d *d, struct bpf_if *bp);
240 static int bpf_detachd(struct bpf_d *d, struct proc *);
241 static void bpf_freed(struct bpf_d *);
242 static int bpf_setif(struct bpf_d *, ifnet_t ifp, bool, bool, bool, struct proc *);
243 static void bpf_timed_out(void *, void *);
244 static void bpf_wakeup(struct bpf_d *);
245 static uint32_t get_pkt_trunc_len(struct bpf_packet *);
246 static void catchpacket(struct bpf_d *, struct bpf_packet *, u_int, int);
247 static void reset_d(struct bpf_d *);
248 static int bpf_setf(struct bpf_d *, u_int, user_addr_t, u_long);
249 static int bpf_getdltlist(struct bpf_d *, caddr_t __bidi_indexable, struct proc *);
250 static int bpf_setdlt(struct bpf_d *, u_int, struct proc *);
251 static int bpf_set_traffic_class(struct bpf_d *, int);
252 static void bpf_set_packet_service_class(struct mbuf *, int);
253
254 static void bpf_acquire_d(struct bpf_d *);
255 static void bpf_release_d(struct bpf_d *);
256
257 static int bpf_devsw_installed;
258
259 void bpf_init(void *unused);
260 static int bpf_tap_callback(struct ifnet *ifp, struct mbuf *m);
261
262 /*
263 * Darwin differs from BSD here, the following are static
264 * on BSD and not static on Darwin.
265 */
266 d_open_t bpfopen;
267 d_close_t bpfclose;
268 d_read_t bpfread;
269 d_write_t bpfwrite;
270 ioctl_fcn_t bpfioctl;
271 select_fcn_t bpfselect;
272
273 /* Darwin's cdevsw struct differs slightly from BSDs */
274 #define CDEV_MAJOR 23
275 static const struct cdevsw bpf_cdevsw = {
276 .d_open = bpfopen,
277 .d_close = bpfclose,
278 .d_read = bpfread,
279 .d_write = bpfwrite,
280 .d_ioctl = bpfioctl,
281 .d_stop = eno_stop,
282 .d_reset = eno_reset,
283 .d_ttys = NULL,
284 .d_select = bpfselect,
285 .d_mmap = eno_mmap,
286 .d_strategy = eno_strat,
287 .d_reserved_1 = eno_getc,
288 .d_reserved_2 = eno_putc,
289 .d_type = 0
290 };
291
292 #define SOCKADDR_HDR_LEN offsetof(struct sockaddr, sa_data)
293
294 static int
bpf_copy_uio_to_mbuf_packet(struct uio * auio,int bytes_to_copy,struct mbuf * top)295 bpf_copy_uio_to_mbuf_packet(struct uio *auio, int bytes_to_copy, struct mbuf *top)
296 {
297 int error = 0;
298
299 for (struct mbuf *m = top; m != NULL; m = m->m_next) {
300 int mlen;
301
302 if (m->m_flags & M_EXT) {
303 mlen = m->m_ext.ext_size - (int)M_LEADINGSPACE(m);
304 } else if (m->m_flags & M_PKTHDR) {
305 mlen = MHLEN - (int)M_LEADINGSPACE(m);
306 } else {
307 mlen = MLEN - (int)M_LEADINGSPACE(m);
308 }
309 int copy_len = imin((int)mlen, bytes_to_copy);
310
311 error = uiomove(mtod(m, caddr_t), (int)copy_len, auio);
312 if (error != 0) {
313 os_log(OS_LOG_DEFAULT, "bpf_copy_uio_to_mbuf_packet: len %d error %d",
314 copy_len, error);
315 goto done;
316 }
317 m->m_len = copy_len;
318 top->m_pkthdr.len += copy_len;
319
320 if (bytes_to_copy > copy_len) {
321 bytes_to_copy -= copy_len;
322 } else {
323 break;
324 }
325 }
326 done:
327 return error;
328 }
329
330 static inline void
bpf_set_bcast_mcast(mbuf_t m,struct ether_header * eh)331 bpf_set_bcast_mcast(mbuf_t m, struct ether_header * eh)
332 {
333 if (ETHER_IS_MULTICAST(eh->ether_dhost)) {
334 if (_ether_cmp(etherbroadcastaddr, eh->ether_dhost) == 0) {
335 m->m_flags |= M_BCAST;
336 } else {
337 m->m_flags |= M_MCAST;
338 }
339 }
340 }
341
342 #if DEBUG | DEVELOPMENT
343 static void
bpf_log_bcast(const char * func,const char * ifname,uint16_t flags,bool hdrcmplt)344 bpf_log_bcast(const char * func, const char * ifname, uint16_t flags,
345 bool hdrcmplt)
346 {
347 const char * type;
348
349 if ((flags & M_BCAST) != 0) {
350 type = "broadcast";
351 } else if ((flags & M_MCAST) != 0) {
352 type = "multicast";
353 } else {
354 type = "unicast";
355 }
356 os_log(OS_LOG_DEFAULT, "%s %s %s hdrcmplt=%s", func, ifname, type,
357 hdrcmplt ? "true" : "false");
358 }
359 #endif /* DEBUG | DEVELOPMENT */
360
361 static int
bpf_movein(struct uio * uio,int copy_len,struct bpf_d * d,struct mbuf ** mp,struct sockaddr * sockp)362 bpf_movein(struct uio *uio, int copy_len, struct bpf_d *d, struct mbuf **mp,
363 struct sockaddr *sockp)
364 {
365 mbuf_ref_t m = NULL;
366 int error;
367 int len;
368 uint8_t sa_family;
369 int hlen = 0;
370 struct ifnet *ifp = d->bd_bif->bif_ifp;
371 int linktype = (int)d->bd_bif->bif_dlt;
372
373 switch (linktype) {
374 #if SLIP
375 case DLT_SLIP:
376 sa_family = AF_INET;
377 hlen = 0;
378 break;
379 #endif /* SLIP */
380
381 case DLT_EN10MB:
382 sa_family = AF_UNSPEC;
383 /* XXX Would MAXLINKHDR be better? */
384 hlen = sizeof(struct ether_header);
385 break;
386
387 #if FDDI
388 case DLT_FDDI:
389 #if defined(__FreeBSD__) || defined(__bsdi__)
390 sa_family = AF_IMPLINK;
391 hlen = 0;
392 #else
393 sa_family = AF_UNSPEC;
394 /* XXX 4(FORMAC)+6(dst)+6(src)+3(LLC)+5(SNAP) */
395 hlen = 24;
396 #endif
397 break;
398 #endif /* FDDI */
399
400 case DLT_RAW:
401 case DLT_NULL:
402 sa_family = AF_UNSPEC;
403 hlen = 0;
404 break;
405
406 #ifdef __FreeBSD__
407 case DLT_ATM_RFC1483:
408 /*
409 * en atm driver requires 4-byte atm pseudo header.
410 * though it isn't standard, vpi:vci needs to be
411 * specified anyway.
412 */
413 sa_family = AF_UNSPEC;
414 hlen = 12; /* XXX 4(ATM_PH) + 3(LLC) + 5(SNAP) */
415 break;
416 #endif
417
418 case DLT_PPP:
419 sa_family = AF_UNSPEC;
420 hlen = 4; /* This should match PPP_HDRLEN */
421 break;
422
423 case DLT_APPLE_IP_OVER_IEEE1394:
424 sa_family = AF_UNSPEC;
425 hlen = sizeof(struct firewire_header);
426 break;
427
428 case DLT_IEEE802_11: /* IEEE 802.11 wireless */
429 sa_family = AF_IEEE80211;
430 hlen = 0;
431 break;
432
433 case DLT_IEEE802_11_RADIO:
434 sa_family = AF_IEEE80211;
435 hlen = 0;
436 break;
437
438 default:
439 return EIO;
440 }
441
442 if (sockp) {
443 /*
444 * Build a sockaddr based on the data link layer type.
445 * We do this at this level because the ethernet header
446 * is copied directly into the data field of the sockaddr.
447 * In the case of SLIP, there is no header and the packet
448 * is forwarded as is.
449 * Also, we are careful to leave room at the front of the mbuf
450 * for the link level header.
451 */
452 if ((hlen + SOCKADDR_HDR_LEN) > sockp->sa_len) {
453 return EIO;
454 }
455 sockp->sa_family = sa_family;
456 } else {
457 /*
458 * We're directly sending the packet data supplied by
459 * the user; we don't need to make room for the link
460 * header, and don't need the header length value any
461 * more, so set it to 0.
462 */
463 hlen = 0;
464 }
465
466 len = (int)uio_resid(uio);
467 if (len < copy_len) {
468 os_log(OS_LOG_DEFAULT, "bpfwrite: len %d if %s less than copy_len %d",
469 (unsigned)len, ifp->if_xname, copy_len);
470 return EMSGSIZE;
471 }
472 len = copy_len;
473 if (len < hlen || (unsigned)len > BPF_WRITE_MAX) {
474 os_log(OS_LOG_DEFAULT, "bpfwrite: bad len %d if %s",
475 (unsigned)len, ifp->if_xname);
476 return EMSGSIZE;
477 }
478 if (d->bd_write_size_max != 0) {
479 if ((len - hlen) > (d->bd_write_size_max + BPF_WRITE_LEEWAY)) {
480 os_log(OS_LOG_DEFAULT, "bpfwrite: len %u - hlen %u too big if %s write_size_max %u",
481 (unsigned)len, (unsigned)hlen, ifp->if_xname, d->bd_write_size_max);
482 }
483 } else if ((len - hlen) > (ifp->if_mtu + BPF_WRITE_LEEWAY)) {
484 os_log(OS_LOG_DEFAULT, "bpfwrite: len %u - hlen %u too big if %s mtu %u",
485 (unsigned)len, (unsigned)hlen, ifp->if_xname, ifp->if_mtu);
486 return EMSGSIZE;
487 }
488
489 /* drop lock while allocating mbuf and copying data */
490 lck_mtx_unlock(bpf_mlock);
491
492 error = mbuf_allocpacket(MBUF_WAITOK, len, NULL, &m);
493 if (error != 0) {
494 os_log(OS_LOG_DEFAULT,
495 "bpfwrite mbuf_allocpacket len %d error %d", len, error);
496 goto bad;
497 }
498 /*
499 * Make room for link header -- the packet length is 0 at this stage
500 */
501 if (hlen != 0) {
502 m->m_data += hlen; /* leading space */
503 error = uiomove((caddr_t)sockp->sa_data, hlen, uio);
504 if (error) {
505 os_log(OS_LOG_DEFAULT,
506 "bpfwrite uiomove hlen %d error %d", hlen, error);
507 goto bad;
508 }
509 len -= hlen;
510 if (linktype == DLT_EN10MB) {
511 struct ether_header * eh;
512
513 eh = (struct ether_header *)(void *)sockp->sa_data;
514 bpf_set_bcast_mcast(m, eh);
515 #if DEBUG || DEVELOPMENT
516 if (__improbable(bpf_debug != 0)) {
517 bpf_log_bcast(__func__, ifp->if_xname,
518 m->m_flags, false);
519 }
520 #endif /* DEBUG || DEVELOPMENT */
521 }
522 }
523 /*
524 * bpf_copy_uio_to_mbuf_packet() does set the length of each mbuf and adds it to
525 * the total packet length
526 */
527 error = bpf_copy_uio_to_mbuf_packet(uio, len, m);
528 if (error != 0) {
529 os_log(OS_LOG_DEFAULT,
530 "bpfwrite bpf_copy_uio_to_mbuf_packet error %d", error);
531 goto bad;
532 }
533
534 /* Check for multicast destination */
535 if (hlen == 0 && linktype == DLT_EN10MB) {
536 struct ether_header *eh;
537
538 eh = mtod(m, struct ether_header *);
539 bpf_set_bcast_mcast(m, eh);
540 #if DEBUG || DEVELOPMENT
541 if (__improbable(bpf_debug != 0)) {
542 bpf_log_bcast(__func__, ifp->if_xname,
543 m->m_flags, true);
544 }
545 #endif /* DEBUG || DEVELOPMENT */
546 }
547 *mp = m;
548
549 lck_mtx_lock(bpf_mlock);
550 return 0;
551 bad:
552 if (m != NULL) {
553 m_freem(m);
554 }
555 lck_mtx_lock(bpf_mlock);
556 return error;
557 }
558
559 static int
bpf_movein_batch(struct uio * uio,struct bpf_d * d,struct mbuf ** mp,struct sockaddr * sockp)560 bpf_movein_batch(struct uio *uio, struct bpf_d *d, struct mbuf **mp,
561 struct sockaddr *sockp)
562 {
563 int error = 0;
564 user_ssize_t resid;
565 int count = 0;
566 struct mbuf *last = NULL;
567
568 *mp = NULL;
569 while ((resid = uio_resid(uio)) >= sizeof(struct bpf_hdr)) {
570 struct bpf_hdr bpfhdr = {};
571 int bpf_hdr_min_len = offsetof(struct bpf_hdr, bh_hdrlen) + sizeof(bpfhdr.bh_hdrlen);
572 int padding_len;
573
574 error = uiomove((caddr_t)&bpfhdr, bpf_hdr_min_len, uio);
575 if (error != 0) {
576 os_log(OS_LOG_DEFAULT, "bpf_movein_batch uiomove error %d", error);
577 break;
578 }
579 /*
580 * Buffer validation:
581 * - ignore bh_tstamp
582 * - bh_hdrlen must fit
583 * - bh_caplen and bh_datalen must be equal
584 */
585 if (bpfhdr.bh_hdrlen < bpf_hdr_min_len) {
586 error = EINVAL;
587 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_hdrlen %u too small",
588 bpfhdr.bh_hdrlen);
589 break;
590 }
591 if (bpfhdr.bh_caplen != bpfhdr.bh_datalen) {
592 error = EINVAL;
593 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_caplen %u != bh_datalen %u",
594 bpfhdr.bh_caplen, bpfhdr.bh_datalen);
595 break;
596 }
597 if (bpfhdr.bh_hdrlen > resid) {
598 error = EINVAL;
599 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bh_hdrlen %u too large",
600 bpfhdr.bh_hdrlen);
601 break;
602 }
603
604 /*
605 * Ignore additional bytes in the header
606 */
607 padding_len = bpfhdr.bh_hdrlen - bpf_hdr_min_len;
608 if (padding_len > 0) {
609 uio_update(uio, padding_len);
610 }
611
612 /* skip empty packets */
613 if (bpfhdr.bh_caplen > 0) {
614 mbuf_ref_t m;
615
616 /*
617 * For time being assume all packets have same destination
618 */
619 error = bpf_movein(uio, bpfhdr.bh_caplen, d, &m, sockp);
620 if (error != 0) {
621 os_log(OS_LOG_DEFAULT, "bpf_movein_batch bpf_movein error %d",
622 error);
623 break;
624 }
625 count += 1;
626
627 if (last == NULL) {
628 *mp = m;
629 } else {
630 last->m_nextpkt = m;
631 }
632 last = m;
633 }
634
635 /*
636 * Each BPF packet is padded for alignment
637 */
638 padding_len = BPF_WORDALIGN(bpfhdr.bh_hdrlen + bpfhdr.bh_caplen) - (bpfhdr.bh_hdrlen + bpfhdr.bh_caplen);
639 if (padding_len > 0) {
640 uio_update(uio, padding_len);
641 }
642 }
643
644 if (error != 0) {
645 if (*mp != NULL) {
646 m_freem_list(*mp);
647 *mp = NULL;
648 }
649 }
650 return error;
651 }
652
653 /*
654 * The dynamic addition of a new device node must block all processes that
655 * are opening the last device so that no process will get an unexpected
656 * ENOENT
657 */
658 static void
bpf_make_dev_t(int maj)659 bpf_make_dev_t(int maj)
660 {
661 static int bpf_growing = 0;
662 unsigned int cur_size = nbpfilter, i;
663
664 if (nbpfilter >= BPF_MAX_DEVICES) {
665 return;
666 }
667
668 while (bpf_growing) {
669 /* Wait until new device has been created */
670 (void) tsleep((caddr_t)&bpf_growing, PZERO, "bpf_growing", 0);
671 }
672 if (nbpfilter > cur_size) {
673 /* other thread grew it already */
674 return;
675 }
676 bpf_growing = 1;
677
678 /* need to grow bpf_dtab first */
679 if (nbpfilter == bpf_dtab_size) {
680 unsigned int new_dtab_size;
681 struct bpf_d **new_dtab = NULL;
682
683 new_dtab_size = bpf_dtab_size + NBPFILTER;
684 new_dtab = krealloc_type(struct bpf_d *,
685 bpf_dtab_size, new_dtab_size, bpf_dtab, Z_WAITOK | Z_ZERO);
686
687 if (new_dtab == 0) {
688 os_log_error(OS_LOG_DEFAULT, "bpf_make_dev_t: malloc bpf_dtab failed");
689 goto done;
690 }
691 bpf_dtab = new_dtab;
692 bpf_dtab_size = new_dtab_size;
693 }
694 i = nbpfilter++;
695 (void) devfs_make_node(makedev(maj, i),
696 DEVFS_CHAR, UID_ROOT, GID_WHEEL, 0600,
697 "bpf%d", i);
698 done:
699 bpf_growing = 0;
700 wakeup((caddr_t)&bpf_growing);
701 }
702
703 /*
704 * Attach file to the bpf interface, i.e. make d listen on bp.
705 */
706 static errno_t
bpf_attachd(struct bpf_d * d,struct bpf_if * bp)707 bpf_attachd(struct bpf_d *d, struct bpf_if *bp)
708 {
709 int first = bp->bif_dlist == NULL;
710 int error = 0;
711
712 /*
713 * Point d at bp, and add d to the interface's list of listeners.
714 * Finally, point the driver's bpf cookie at the interface so
715 * it will divert packets to bpf.
716 */
717 d->bd_bif = bp;
718 d->bd_next = bp->bif_dlist;
719 bp->bif_dlist = d;
720 bpf_bpfd_cnt++;
721
722 /*
723 * Take a reference on the device even if an error is returned
724 * because we keep the device in the interface's list of listeners
725 */
726 bpf_acquire_d(d);
727
728 if (first) {
729 /* Find the default bpf entry for this ifp */
730 if (bp->bif_ifp->if_bpf == NULL) {
731 struct bpf_if *tmp, *primary = NULL;
732
733 for (tmp = bpf_iflist; tmp; tmp = tmp->bif_next) {
734 if (tmp->bif_ifp == bp->bif_ifp) {
735 primary = tmp;
736 break;
737 }
738 }
739 bp->bif_ifp->if_bpf = primary;
740 }
741 /* Only call dlil_set_bpf_tap for primary dlt */
742 if (bp->bif_ifp->if_bpf == bp) {
743 dlil_set_bpf_tap(bp->bif_ifp, BPF_TAP_INPUT_OUTPUT,
744 bpf_tap_callback);
745 }
746
747 if (bp->bif_tap != NULL) {
748 error = bp->bif_tap(bp->bif_ifp, bp->bif_dlt,
749 BPF_TAP_INPUT_OUTPUT);
750 }
751 }
752
753 /*
754 * Reset the detach flags in case we previously detached an interface
755 */
756 d->bd_flags &= ~(BPF_DETACHING | BPF_DETACHED);
757
758 if (bp->bif_dlt == DLT_PKTAP) {
759 d->bd_flags |= BPF_FINALIZE_PKTAP;
760 } else {
761 d->bd_flags &= ~BPF_FINALIZE_PKTAP;
762 }
763 return error;
764 }
765
766 /*
767 * Detach a file from its interface.
768 *
769 * Return 1 if was closed by some thread, 0 otherwise
770 */
771 static int
bpf_detachd(struct bpf_d * d,struct proc * proc)772 bpf_detachd(struct bpf_d *d, struct proc *proc)
773 {
774 struct bpf_d **p;
775 struct bpf_if *bp;
776 struct ifnet *ifp;
777 uint32_t dlt;
778 bpf_tap_func disable_tap;
779 uint8_t bd_promisc;
780
781 int bpf_closed = d->bd_flags & BPF_CLOSING;
782 /*
783 * Some other thread already detached
784 */
785 if ((d->bd_flags & (BPF_DETACHED | BPF_DETACHING)) != 0) {
786 goto done;
787 }
788 /*
789 * This thread is doing the detach
790 */
791 d->bd_flags |= BPF_DETACHING;
792
793 ifp = d->bd_bif->bif_ifp;
794 bp = d->bd_bif;
795
796 /* Remove d from the interface's descriptor list. */
797 p = &bp->bif_dlist;
798 while (*p != d) {
799 p = &(*p)->bd_next;
800 if (*p == 0) {
801 panic("bpf_detachd: descriptor not in list");
802 }
803 }
804 *p = (*p)->bd_next;
805 bpf_bpfd_cnt--;
806 disable_tap = NULL;
807 if (bp->bif_dlist == 0) {
808 /*
809 * Let the driver know that there are no more listeners.
810 */
811 /* Only call dlil_set_bpf_tap for primary dlt */
812 if (bp->bif_ifp->if_bpf == bp) {
813 dlil_set_bpf_tap(ifp, BPF_TAP_DISABLE, NULL);
814 }
815
816 disable_tap = bp->bif_tap;
817 if (disable_tap) {
818 dlt = bp->bif_dlt;
819 }
820
821 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
822 if (bp->bif_ifp == ifp && bp->bif_dlist != 0) {
823 break;
824 }
825 }
826 if (bp == NULL) {
827 ifp->if_bpf = NULL;
828 }
829 }
830 d->bd_bif = NULL;
831
832 /*
833 * Stop disabling input
834 */
835 if ((d->bd_flags & BPF_DIVERT_IN) != 0) {
836 if_clear_xflags(ifp, IFXF_DISABLE_INPUT);
837 d->bd_flags &= ~BPF_DIVERT_IN;
838
839 os_log(OS_LOG_DEFAULT,
840 "bpf_detachd: bpf%d %s disable input 0",
841 d->bd_dev_minor, if_name(ifp));
842 }
843
844 /*
845 * Check if this descriptor had requested promiscuous mode.
846 * If so, turn it off.
847 */
848 bd_promisc = d->bd_promisc;
849 d->bd_promisc = 0;
850
851 lck_mtx_unlock(bpf_mlock);
852 if (bd_promisc) {
853 if (ifnet_set_promiscuous(ifp, 0)) {
854 /*
855 * Something is really wrong if we were able to put
856 * the driver into promiscuous mode, but can't
857 * take it out.
858 * Most likely the network interface is gone.
859 */
860 os_log_error(OS_LOG_DEFAULT,
861 "%s: bpf%d ifnet_set_promiscuous %s failed",
862 __func__, d->bd_dev_minor, if_name(ifp));
863 }
864 }
865
866 if (disable_tap) {
867 disable_tap(ifp, dlt, BPF_TAP_DISABLE);
868 }
869 lck_mtx_lock(bpf_mlock);
870
871 /*
872 * Wake up other thread that are waiting for this thread to finish
873 * detaching
874 */
875 d->bd_flags &= ~BPF_DETACHING;
876 d->bd_flags |= BPF_DETACHED;
877
878 /* Refresh the local variable as d could have been modified */
879 bpf_closed = d->bd_flags & BPF_CLOSING;
880
881 os_log(OS_LOG_DEFAULT, "bpf%d%s detached from %s fcount %llu dcount %llu by %s:%u",
882 d->bd_dev_minor, bpf_closed ? " closed and" : "", if_name(ifp),
883 d->bd_fcount, d->bd_dcount, proc_name_address(proc), proc_pid(proc));
884
885 /*
886 * Note that We've kept the reference because we may have dropped
887 * the lock when turning off promiscuous mode
888 */
889 bpf_release_d(d);
890 done:
891 /*
892 * Let the caller know the bpf_d is closed
893 */
894 if (bpf_closed) {
895 return 1;
896 } else {
897 return 0;
898 }
899 }
900
901 /*
902 * Start asynchronous timer, if necessary.
903 * Must be called with bpf_mlock held.
904 */
905 static void
bpf_start_timer(struct bpf_d * d)906 bpf_start_timer(struct bpf_d *d)
907 {
908 uint64_t deadline;
909 struct timeval tv;
910
911 if (d->bd_rtout > 0 && d->bd_state == BPF_IDLE) {
912 tv.tv_sec = d->bd_rtout / hz;
913 tv.tv_usec = (d->bd_rtout % hz) * tick;
914
915 clock_interval_to_deadline(
916 (uint32_t)tv.tv_sec * USEC_PER_SEC + tv.tv_usec,
917 NSEC_PER_USEC, &deadline);
918 /*
919 * The state is BPF_IDLE, so the timer hasn't
920 * been started yet, and hasn't gone off yet;
921 * there is no thread call scheduled, so this
922 * won't change the schedule.
923 *
924 * XXX - what if, by the time it gets entered,
925 * the deadline has already passed?
926 */
927 thread_call_enter_delayed(d->bd_thread_call, deadline);
928 d->bd_state = BPF_WAITING;
929 }
930 }
931
932 /*
933 * Cancel asynchronous timer.
934 * Must be called with bpf_mlock held.
935 */
936 static boolean_t
bpf_stop_timer(struct bpf_d * d)937 bpf_stop_timer(struct bpf_d *d)
938 {
939 /*
940 * If the timer has already gone off, this does nothing.
941 * Our caller is expected to set d->bd_state to BPF_IDLE,
942 * with the bpf_mlock, after we are called. bpf_timed_out()
943 * also grabs bpf_mlock, so, if the timer has gone off and
944 * bpf_timed_out() hasn't finished, it's waiting for the
945 * lock; when this thread releases the lock, it will
946 * find the state is BPF_IDLE, and just release the
947 * lock and return.
948 */
949 return thread_call_cancel(d->bd_thread_call);
950 }
951
952 void
bpf_acquire_d(struct bpf_d * d)953 bpf_acquire_d(struct bpf_d *d)
954 {
955 void *__single lr_saved = __unsafe_forge_single(void *, __builtin_return_address(0));
956
957 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
958
959 d->bd_refcnt += 1;
960
961 d->bd_ref_lr[d->bd_next_ref_lr] = lr_saved;
962 d->bd_next_ref_lr = (d->bd_next_ref_lr + 1) % BPF_REF_HIST;
963 }
964
965 void
bpf_release_d(struct bpf_d * d)966 bpf_release_d(struct bpf_d *d)
967 {
968 void *__single lr_saved = __unsafe_forge_single(void *, __builtin_return_address(0));
969
970 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
971
972 if (d->bd_refcnt <= 0) {
973 panic("%s: %p refcnt <= 0", __func__, d);
974 }
975
976 d->bd_refcnt -= 1;
977
978 d->bd_unref_lr[d->bd_next_unref_lr] = lr_saved;
979 d->bd_next_unref_lr = (d->bd_next_unref_lr + 1) % BPF_REF_HIST;
980
981 if (d->bd_refcnt == 0) {
982 /* Assert the device is detached */
983 if ((d->bd_flags & BPF_DETACHED) == 0) {
984 panic("%s: %p BPF_DETACHED not set", __func__, d);
985 }
986
987 kfree_type(struct bpf_d, d);
988 }
989 }
990
991 /*
992 * Open ethernet device. Returns ENXIO for illegal minor device number,
993 * EBUSY if file is open by another process.
994 */
995 /* ARGSUSED */
996 int
bpfopen(dev_t dev,int flags,__unused int fmt,struct proc * proc)997 bpfopen(dev_t dev, int flags, __unused int fmt,
998 struct proc *proc)
999 {
1000 struct bpf_d *d;
1001
1002 lck_mtx_lock(bpf_mlock);
1003 if ((unsigned int) minor(dev) >= nbpfilter) {
1004 lck_mtx_unlock(bpf_mlock);
1005 return ENXIO;
1006 }
1007 /*
1008 * New device nodes are created on demand when opening the last one.
1009 * The programming model is for processes to loop on the minor starting
1010 * at 0 as long as EBUSY is returned. The loop stops when either the
1011 * open succeeds or an error other that EBUSY is returned. That means
1012 * that bpf_make_dev_t() must block all processes that are opening the
1013 * last node. If not all processes are blocked, they could unexpectedly
1014 * get ENOENT and abort their opening loop.
1015 */
1016 if ((unsigned int) minor(dev) == (nbpfilter - 1)) {
1017 bpf_make_dev_t(major(dev));
1018 }
1019
1020 /*
1021 * Each minor can be opened by only one process. If the requested
1022 * minor is in use, return EBUSY.
1023 *
1024 * Important: bpfopen() and bpfclose() have to check and set the status
1025 * of a device in the same lockin context otherwise the device may be
1026 * leaked because the vnode use count will be unpextectly greater than 1
1027 * when close() is called.
1028 */
1029 if (bpf_dtab[minor(dev)] == NULL) {
1030 /* Reserve while opening */
1031 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED;
1032 } else {
1033 lck_mtx_unlock(bpf_mlock);
1034 return EBUSY;
1035 }
1036 d = kalloc_type(struct bpf_d, Z_WAITOK | Z_ZERO);
1037 if (d == NULL) {
1038 /* this really is a catastrophic failure */
1039 os_log_error(OS_LOG_DEFAULT,
1040 "bpfopen: bpf%d kalloc_type bpf_d failed", minor(dev));
1041 bpf_dtab[minor(dev)] = NULL;
1042 lck_mtx_unlock(bpf_mlock);
1043 return ENOMEM;
1044 }
1045
1046 /* Mark "in use" and do most initialization. */
1047 bpf_acquire_d(d);
1048 d->bd_bufsize = bpf_bufsize;
1049 d->bd_sig = SIGIO;
1050 d->bd_direction = BPF_D_INOUT;
1051 d->bd_oflags = flags;
1052 d->bd_state = BPF_IDLE;
1053 d->bd_traffic_class = SO_TC_BE;
1054 d->bd_flags |= BPF_DETACHED;
1055 if (bpf_wantpktap) {
1056 d->bd_flags |= BPF_WANT_PKTAP;
1057 } else {
1058 d->bd_flags &= ~BPF_WANT_PKTAP;
1059 }
1060
1061 d->bd_thread_call = thread_call_allocate(bpf_timed_out, d);
1062 if (d->bd_thread_call == NULL) {
1063 os_log_error(OS_LOG_DEFAULT, "bpfopen: bpf%d malloc thread call failed",
1064 minor(dev));
1065 bpf_dtab[minor(dev)] = NULL;
1066 bpf_release_d(d);
1067 lck_mtx_unlock(bpf_mlock);
1068
1069 return ENOMEM;
1070 }
1071
1072 /* Use the proc pointer for comparaison so no need to take a reference */
1073 d->bd_opened_by = proc;
1074 uuid_generate(d->bd_uuid);
1075 d->bd_pid = proc_pid(proc);
1076
1077 d->bd_dev_minor = minor(dev);
1078 bpf_dtab[minor(dev)] = d; /* Mark opened */
1079 lck_mtx_unlock(bpf_mlock);
1080
1081 if (bpf_debug) {
1082 os_log(OS_LOG_DEFAULT, "bpf%u opened by %s.%u",
1083 d->bd_dev_minor, proc_name_address(proc), d->bd_pid);
1084 }
1085 return 0;
1086 }
1087
1088 /*
1089 * Close the descriptor by detaching it from its interface,
1090 * deallocating its buffers, and marking it free.
1091 */
1092 /* ARGSUSED */
1093 int
bpfclose(dev_t dev,__unused int flags,__unused int fmt,struct proc * proc)1094 bpfclose(dev_t dev, __unused int flags, __unused int fmt,
1095 struct proc *proc)
1096 {
1097 struct bpf_d *d;
1098
1099 /* Take BPF lock to ensure no other thread is using the device */
1100 lck_mtx_lock(bpf_mlock);
1101
1102 d = bpf_dtab[minor(dev)];
1103 if (d == NULL || d == BPF_DEV_RESERVED) {
1104 lck_mtx_unlock(bpf_mlock);
1105 return ENXIO;
1106 }
1107
1108 /*
1109 * Other threads may call bpd_detachd() if we drop the bpf_mlock
1110 */
1111 d->bd_flags |= BPF_CLOSING;
1112
1113 if (bpf_debug != 0) {
1114 os_log(OS_LOG_DEFAULT, "%s: bpf%d",
1115 __func__, d->bd_dev_minor);
1116 }
1117
1118 bpf_dtab[minor(dev)] = BPF_DEV_RESERVED; /* Reserve while closing */
1119
1120 /*
1121 * Deal with any in-progress timeouts.
1122 */
1123 switch (d->bd_state) {
1124 case BPF_IDLE:
1125 /*
1126 * Not waiting for a timeout, and no timeout happened.
1127 */
1128 break;
1129
1130 case BPF_WAITING:
1131 /*
1132 * Waiting for a timeout.
1133 * Cancel any timer that has yet to go off,
1134 * and mark the state as "closing".
1135 * Then drop the lock to allow any timers that
1136 * *have* gone off to run to completion, and wait
1137 * for them to finish.
1138 */
1139 if (!bpf_stop_timer(d)) {
1140 /*
1141 * There was no pending call, so the call must
1142 * have been in progress. Wait for the call to
1143 * complete; we have to drop the lock while
1144 * waiting. to let the in-progrss call complete
1145 */
1146 d->bd_state = BPF_DRAINING;
1147 while (d->bd_state == BPF_DRAINING) {
1148 msleep((caddr_t)d, bpf_mlock, PRINET,
1149 "bpfdraining", NULL);
1150 }
1151 }
1152 d->bd_state = BPF_IDLE;
1153 break;
1154
1155 case BPF_TIMED_OUT:
1156 /*
1157 * Timer went off, and the timeout routine finished.
1158 */
1159 d->bd_state = BPF_IDLE;
1160 break;
1161
1162 case BPF_DRAINING:
1163 /*
1164 * Another thread is blocked on a close waiting for
1165 * a timeout to finish.
1166 * This "shouldn't happen", as the first thread to enter
1167 * bpfclose() will set bpf_dtab[minor(dev)] to 1, and
1168 * all subsequent threads should see that and fail with
1169 * ENXIO.
1170 */
1171 panic("Two threads blocked in a BPF close");
1172 break;
1173 }
1174
1175 if (d->bd_bif) {
1176 bpf_detachd(d, proc);
1177 }
1178 selthreadclear(&d->bd_sel);
1179 thread_call_free(d->bd_thread_call);
1180
1181 while (d->bd_hbuf_read || d->bd_hbuf_write) {
1182 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfclose", NULL);
1183 }
1184
1185 if (bpf_debug) {
1186 os_log(OS_LOG_DEFAULT,
1187 "bpf%u closed by %s.%u dcount %llu fcount %llu ccount %llu",
1188 d->bd_dev_minor, proc_name_address(proc), proc_pid(proc),
1189 d->bd_dcount, d->bd_fcount, d->bd_bcs.bcs_count_compressed_prefix);
1190 }
1191
1192 bpf_freed(d);
1193
1194 /* Mark free in same context as bpfopen comes to check */
1195 bpf_dtab[minor(dev)] = NULL; /* Mark closed */
1196
1197 bpf_release_d(d);
1198
1199 lck_mtx_unlock(bpf_mlock);
1200
1201 return 0;
1202 }
1203
1204 #define BPF_SLEEP bpf_sleep
1205
1206 static int
bpf_sleep(struct bpf_d * d,int pri,const char * wmesg,int timo)1207 bpf_sleep(struct bpf_d *d, int pri, const char *wmesg, int timo)
1208 {
1209 u_int64_t abstime = 0;
1210
1211 if (timo != 0) {
1212 clock_interval_to_deadline(timo, NSEC_PER_SEC / hz, &abstime);
1213 }
1214
1215 return msleep1((caddr_t)d, bpf_mlock, pri, wmesg, abstime);
1216 }
1217
1218 static void
bpf_finalize_pktap(struct bpf_hdr * hp,struct pktap_header * pktaphdr)1219 bpf_finalize_pktap(struct bpf_hdr *hp, struct pktap_header *pktaphdr)
1220 {
1221 if (pktaphdr->pth_flags & PTH_FLAG_V2_HDR) {
1222 struct pktap_v2_hdr *pktap_v2_hdr;
1223
1224 pktap_v2_hdr = (struct pktap_v2_hdr *)pktaphdr;
1225
1226 if (pktap_v2_hdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1227 pktap_v2_finalize_proc_info(pktap_v2_hdr);
1228 }
1229 } else {
1230 if (pktaphdr->pth_flags & PTH_FLAG_DELAY_PKTAP) {
1231 pktap_finalize_proc_info(pktaphdr);
1232 }
1233
1234 if (pktaphdr->pth_flags & PTH_FLAG_TSTAMP) {
1235 hp->bh_tstamp.tv_sec = pktaphdr->pth_tstamp.tv_sec;
1236 hp->bh_tstamp.tv_usec = pktaphdr->pth_tstamp.tv_usec;
1237 }
1238 }
1239 }
1240
1241 /*
1242 * Rotate the packet buffers in descriptor d. Move the store buffer
1243 * into the hold slot, and the free buffer into the store slot.
1244 * Zero the length of the new store buffer.
1245 *
1246 * Note: in head drop mode, the hold buffer can be dropped so the fist packet of the
1247 * store buffer cannot be compressed as it otherwise would refer to deleted data
1248 * in a dropped hold buffer that the reader process does know about
1249 */
1250 #define ROTATE_BUFFERS(d) do { \
1251 if (d->bd_hbuf_read) \
1252 panic("rotating bpf buffers during read"); \
1253 (d)->bd_hbuf = (d)->bd_sbuf; \
1254 (d)->bd_hlen = (d)->bd_slen; \
1255 (d)->bd_hcnt = (d)->bd_scnt; \
1256 (d)->bd_sbuf = (d)->bd_fbuf; \
1257 (d)->bd_slen = 0; \
1258 (d)->bd_scnt = 0; \
1259 (d)->bd_fbuf = NULL; \
1260 if ((d)->bd_headdrop != 0) \
1261 (d)->bd_prev_slen = 0; \
1262 } while(false)
1263
1264 /*
1265 * bpfread - read next chunk of packets from buffers
1266 */
1267 int
bpfread(dev_t dev,struct uio * uio,int ioflag)1268 bpfread(dev_t dev, struct uio *uio, int ioflag)
1269 {
1270 struct bpf_d *d;
1271 caddr_t hbuf;
1272 int timed_out, hbuf_len;
1273 int error;
1274 int flags;
1275
1276 lck_mtx_lock(bpf_mlock);
1277
1278 d = bpf_dtab[minor(dev)];
1279 if (d == NULL || d == BPF_DEV_RESERVED ||
1280 (d->bd_flags & BPF_CLOSING) != 0) {
1281 lck_mtx_unlock(bpf_mlock);
1282 return ENXIO;
1283 }
1284
1285 bpf_acquire_d(d);
1286
1287 /*
1288 * Restrict application to use a buffer the same size as
1289 * as kernel buffers.
1290 */
1291 if (uio_resid(uio) != d->bd_bufsize) {
1292 bpf_release_d(d);
1293 lck_mtx_unlock(bpf_mlock);
1294 return EINVAL;
1295 }
1296
1297 if (d->bd_state == BPF_WAITING) {
1298 bpf_stop_timer(d);
1299 }
1300
1301 timed_out = (d->bd_state == BPF_TIMED_OUT);
1302 d->bd_state = BPF_IDLE;
1303
1304 while (d->bd_hbuf_read) {
1305 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfread", NULL);
1306 }
1307
1308 if ((d->bd_flags & BPF_CLOSING) != 0) {
1309 bpf_release_d(d);
1310 lck_mtx_unlock(bpf_mlock);
1311 return ENXIO;
1312 }
1313 /*
1314 * If the hold buffer is empty, then do a timed sleep, which
1315 * ends when the timeout expires or when enough packets
1316 * have arrived to fill the store buffer.
1317 */
1318 while (d->bd_hbuf == 0) {
1319 if ((d->bd_immediate || timed_out || (ioflag & IO_NDELAY)) &&
1320 d->bd_slen != 0) {
1321 /*
1322 * We're in immediate mode, or are reading
1323 * in non-blocking mode, or a timer was
1324 * started before the read (e.g., by select()
1325 * or poll()) and has expired and a packet(s)
1326 * either arrived since the previous
1327 * read or arrived while we were asleep.
1328 * Rotate the buffers and return what's here.
1329 */
1330 ROTATE_BUFFERS(d);
1331 break;
1332 }
1333
1334 /*
1335 * No data is available, check to see if the bpf device
1336 * is still pointed at a real interface. If not, return
1337 * ENXIO so that the userland process knows to rebind
1338 * it before using it again.
1339 */
1340 if (d->bd_bif == NULL) {
1341 bpf_release_d(d);
1342 lck_mtx_unlock(bpf_mlock);
1343 return ENXIO;
1344 }
1345 if (ioflag & IO_NDELAY) {
1346 bpf_release_d(d);
1347 lck_mtx_unlock(bpf_mlock);
1348 return EWOULDBLOCK;
1349 }
1350 error = BPF_SLEEP(d, PRINET | PCATCH, "bpf", d->bd_rtout);
1351 /*
1352 * Make sure device is still opened
1353 */
1354 if ((d->bd_flags & BPF_CLOSING) != 0) {
1355 bpf_release_d(d);
1356 lck_mtx_unlock(bpf_mlock);
1357 return ENXIO;
1358 }
1359
1360 while (d->bd_hbuf_read) {
1361 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_read",
1362 NULL);
1363 }
1364
1365 if ((d->bd_flags & BPF_CLOSING) != 0) {
1366 bpf_release_d(d);
1367 lck_mtx_unlock(bpf_mlock);
1368 return ENXIO;
1369 }
1370
1371 if (error == EINTR || error == ERESTART) {
1372 if (d->bd_hbuf != NULL) {
1373 /*
1374 * Because we msleep, the hold buffer might
1375 * be filled when we wake up. Avoid rotating
1376 * in this case.
1377 */
1378 break;
1379 }
1380 if (d->bd_slen != 0) {
1381 /*
1382 * Sometimes we may be interrupted often and
1383 * the sleep above will not timeout.
1384 * Regardless, we should rotate the buffers
1385 * if there's any new data pending and
1386 * return it.
1387 */
1388 ROTATE_BUFFERS(d);
1389 break;
1390 }
1391 bpf_release_d(d);
1392 lck_mtx_unlock(bpf_mlock);
1393 if (error == ERESTART) {
1394 os_log(OS_LOG_DEFAULT, "%s: bpf%d ERESTART to EINTR",
1395 __func__, d->bd_dev_minor);
1396 error = EINTR;
1397 }
1398 return error;
1399 }
1400 if (error == EWOULDBLOCK) {
1401 /*
1402 * On a timeout, return what's in the buffer,
1403 * which may be nothing. If there is something
1404 * in the store buffer, we can rotate the buffers.
1405 */
1406 if (d->bd_hbuf) {
1407 /*
1408 * We filled up the buffer in between
1409 * getting the timeout and arriving
1410 * here, so we don't need to rotate.
1411 */
1412 break;
1413 }
1414
1415 if (d->bd_slen == 0) {
1416 bpf_release_d(d);
1417 lck_mtx_unlock(bpf_mlock);
1418 return 0;
1419 }
1420 ROTATE_BUFFERS(d);
1421 break;
1422 }
1423 }
1424 /*
1425 * At this point, we know we have something in the hold slot.
1426 */
1427
1428 /*
1429 * Set the hold buffer read. So we do not
1430 * rotate the buffers until the hold buffer
1431 * read is complete. Also to avoid issues resulting
1432 * from page faults during disk sleep (<rdar://problem/13436396>).
1433 */
1434 d->bd_hbuf_read = true;
1435 hbuf = d->bd_hbuf;
1436 hbuf_len = d->bd_hlen;
1437 flags = d->bd_flags;
1438 d->bd_bcs.bcs_total_read += d->bd_hcnt;
1439 lck_mtx_unlock(bpf_mlock);
1440
1441 /*
1442 * Before we move data to userland, we fill out the extended
1443 * header fields.
1444 */
1445 if (flags & BPF_EXTENDED_HDR) {
1446 char *p;
1447
1448 p = hbuf;
1449 while (p < hbuf + hbuf_len) {
1450 struct bpf_hdr_ext *ehp;
1451 uint32_t flowid;
1452 struct so_procinfo soprocinfo;
1453 int found = 0;
1454
1455 ehp = (struct bpf_hdr_ext *)(void *)p;
1456 if ((flowid = ehp->bh_flowid) != 0) {
1457 if (ehp->bh_flags & BPF_HDR_EXT_FLAGS_TCP) {
1458 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_TCP;
1459 found = inp_findinpcb_procinfo(&tcbinfo,
1460 flowid, &soprocinfo);
1461 } else if (ehp->bh_flags == BPF_HDR_EXT_FLAGS_UDP) {
1462 ehp->bh_flags &= ~BPF_HDR_EXT_FLAGS_UDP;
1463 found = inp_findinpcb_procinfo(&udbinfo,
1464 flowid, &soprocinfo);
1465 }
1466 if (found == 1) {
1467 ehp->bh_pid = soprocinfo.spi_pid;
1468 strbufcpy(ehp->bh_comm,
1469 soprocinfo.spi_proc_name);
1470 }
1471 ehp->bh_flowid = 0;
1472 }
1473
1474 if ((flags & BPF_FINALIZE_PKTAP) != 0 && ehp->bh_complen == 0) {
1475 struct pktap_header *pktaphdr;
1476
1477 pktaphdr = (struct pktap_header *)(void *)
1478 (p + BPF_WORDALIGN(ehp->bh_hdrlen));
1479
1480 bpf_finalize_pktap((struct bpf_hdr *) ehp,
1481 pktaphdr);
1482 }
1483 p += BPF_WORDALIGN(ehp->bh_hdrlen + ehp->bh_caplen);
1484 }
1485 } else if (flags & BPF_FINALIZE_PKTAP) {
1486 char *p;
1487
1488 p = hbuf;
1489
1490 while (p < hbuf + hbuf_len) {
1491 struct bpf_hdr *hp;
1492 struct pktap_header *pktaphdr;
1493
1494 hp = (struct bpf_hdr *)(void *)p;
1495
1496 /*
1497 * Cannot finalize a compressed pktap header as we may not have
1498 * all the fields present
1499 */
1500 if (d->bd_flags & BPF_COMP_ENABLED) {
1501 struct bpf_comp_hdr *hcp;
1502
1503 hcp = (struct bpf_comp_hdr *)(void *)p;
1504
1505 if (hcp->bh_complen != 0) {
1506 p += BPF_WORDALIGN(hcp->bh_hdrlen + hcp->bh_caplen);
1507 continue;
1508 }
1509 }
1510
1511 pktaphdr = (struct pktap_header *)(void *)
1512 (p + BPF_WORDALIGN(hp->bh_hdrlen));
1513
1514 bpf_finalize_pktap(hp, pktaphdr);
1515
1516 p += BPF_WORDALIGN(hp->bh_hdrlen + hp->bh_caplen);
1517 }
1518 }
1519
1520 /*
1521 * Move data from hold buffer into user space.
1522 * We know the entire buffer is transferred since
1523 * we checked above that the read buffer is bpf_bufsize bytes.
1524 */
1525 error = uiomove(hbuf, hbuf_len, uio);
1526
1527 lck_mtx_lock(bpf_mlock);
1528 /*
1529 * Make sure device is still opened
1530 */
1531 if ((d->bd_flags & BPF_CLOSING) != 0) {
1532 bpf_release_d(d);
1533 lck_mtx_unlock(bpf_mlock);
1534 return ENXIO;
1535 }
1536
1537 d->bd_hbuf_read = false;
1538 d->bd_fbuf = d->bd_hbuf;
1539 d->bd_hbuf = NULL;
1540 d->bd_hlen = 0;
1541 d->bd_hcnt = 0;
1542 wakeup((caddr_t)d);
1543
1544 bpf_release_d(d);
1545 lck_mtx_unlock(bpf_mlock);
1546 return error;
1547 }
1548
1549 /*
1550 * If there are processes sleeping on this descriptor, wake them up.
1551 */
1552 static void
bpf_wakeup(struct bpf_d * d)1553 bpf_wakeup(struct bpf_d *d)
1554 {
1555 if (d->bd_state == BPF_WAITING) {
1556 bpf_stop_timer(d);
1557 d->bd_state = BPF_IDLE;
1558 }
1559 wakeup((caddr_t)d);
1560 if (d->bd_async && d->bd_sig && d->bd_sigio) {
1561 pgsigio(d->bd_sigio, d->bd_sig);
1562 }
1563
1564 selwakeup(&d->bd_sel);
1565 if ((d->bd_flags & BPF_KNOTE)) {
1566 KNOTE(&d->bd_sel.si_note, 1);
1567 }
1568 }
1569
1570 static void
bpf_timed_out(void * arg,__unused void * dummy)1571 bpf_timed_out(void *arg, __unused void *dummy)
1572 {
1573 struct bpf_d *d = (struct bpf_d *)arg;
1574
1575 lck_mtx_lock(bpf_mlock);
1576 if (d->bd_state == BPF_WAITING) {
1577 /*
1578 * There's a select or kqueue waiting for this; if there's
1579 * now stuff to read, wake it up.
1580 */
1581 d->bd_state = BPF_TIMED_OUT;
1582 if (d->bd_slen != 0) {
1583 bpf_wakeup(d);
1584 }
1585 } else if (d->bd_state == BPF_DRAINING) {
1586 /*
1587 * A close is waiting for this to finish.
1588 * Mark it as finished, and wake the close up.
1589 */
1590 d->bd_state = BPF_IDLE;
1591 bpf_wakeup(d);
1592 }
1593 lck_mtx_unlock(bpf_mlock);
1594 }
1595
1596 /* keep in sync with bpf_movein above: */
1597 #define MAX_DATALINK_HDR_LEN (sizeof(struct firewire_header))
1598
1599 int
bpfwrite(dev_t dev,struct uio * uio,__unused int ioflag)1600 bpfwrite(dev_t dev, struct uio *uio, __unused int ioflag)
1601 {
1602 struct bpf_d *d;
1603 struct ifnet *ifp;
1604 mbuf_ref_t m = NULL;
1605 int error = 0;
1606 char dst_buf[SOCKADDR_HDR_LEN + MAX_DATALINK_HDR_LEN] = {};
1607 int bif_dlt;
1608 int bd_hdrcmplt;
1609 bpf_send_func bif_send;
1610
1611 lck_mtx_lock(bpf_mlock);
1612
1613 while (true) {
1614 d = bpf_dtab[minor(dev)];
1615 if (d == NULL || d == BPF_DEV_RESERVED ||
1616 (d->bd_flags & BPF_CLOSING) != 0) {
1617 lck_mtx_unlock(bpf_mlock);
1618 return ENXIO;
1619 }
1620
1621 if (d->bd_hbuf_write) {
1622 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfwrite",
1623 NULL);
1624 } else {
1625 break;
1626 }
1627 }
1628 d->bd_hbuf_write = true;
1629
1630 bpf_acquire_d(d);
1631
1632 ++d->bd_wcount;
1633
1634 if (d->bd_bif == NULL) {
1635 error = ENXIO;
1636 goto done;
1637 }
1638
1639 ifp = d->bd_bif->bif_ifp;
1640
1641 if (IFNET_IS_MANAGEMENT(ifp) &&
1642 IOCurrentTaskHasEntitlement(MANAGEMENT_DATA_ENTITLEMENT) == false) {
1643 ++d->bd_wdcount;
1644 bpf_release_d(d);
1645 lck_mtx_unlock(bpf_mlock);
1646 return ENETDOWN;
1647 }
1648
1649 if ((ifp->if_flags & IFF_UP) == 0) {
1650 error = ENETDOWN;
1651 goto done;
1652 }
1653 int resid = (int)uio_resid(uio);
1654 if (resid <= 0) {
1655 error = resid == 0 ? 0 : EINVAL;
1656 os_log(OS_LOG_DEFAULT, "bpfwrite: resid %d error %d", resid, error);
1657 goto done;
1658 }
1659 SA(dst_buf)->sa_len = sizeof(dst_buf);
1660
1661 /*
1662 * geting variables onto stack before dropping the lock
1663 */
1664 bif_dlt = (int)d->bd_bif->bif_dlt;
1665 bd_hdrcmplt = d->bd_hdrcmplt;
1666 bool batch_write = (d->bd_flags & BPF_BATCH_WRITE) ? true : false;
1667
1668 if (batch_write) {
1669 error = bpf_movein_batch(uio, d, &m, bd_hdrcmplt ? NULL : SA(dst_buf));
1670 if (error != 0) {
1671 goto done;
1672 }
1673 } else {
1674 error = bpf_movein(uio, resid, d, &m, bd_hdrcmplt ? NULL : SA(dst_buf));
1675 if (error != 0) {
1676 goto done;
1677 }
1678 bpf_set_packet_service_class(m, d->bd_traffic_class);
1679 }
1680
1681 /* verify the device is still open */
1682 if ((d->bd_flags & BPF_CLOSING) != 0) {
1683 error = ENXIO;
1684 goto done;
1685 }
1686
1687 if (d->bd_bif == NULL || d->bd_bif->bif_ifp != ifp) {
1688 error = ENXIO;
1689 goto done;
1690 }
1691
1692 bif_send = d->bd_bif->bif_send;
1693
1694 lck_mtx_unlock(bpf_mlock);
1695
1696 if (bd_hdrcmplt) {
1697 if (bif_send) {
1698 /*
1699 * Send one packet at a time, the driver frees the mbuf
1700 * but we need to take care of the leftover
1701 */
1702 while (m != NULL && error == 0) {
1703 struct mbuf *next = m->m_nextpkt;
1704
1705 m->m_nextpkt = NULL;
1706 error = bif_send(ifp, bif_dlt, m);
1707 m = next;
1708 }
1709 } else {
1710 error = dlil_output(ifp, 0, m, NULL, NULL,
1711 DLIL_OUTPUT_FLAGS_RAW, NULL);
1712 /* Make sure we do not double free */
1713 m = NULL;
1714 }
1715 } else {
1716 error = dlil_output(ifp, PF_INET, m, NULL, SA(dst_buf),
1717 DLIL_OUTPUT_FLAGS_NONE, NULL);
1718 /* Make sure we do not double free */
1719 m = NULL;
1720 }
1721
1722 lck_mtx_lock(bpf_mlock);
1723 done:
1724 if (error != 0 && m != NULL) {
1725 ++d->bd_wdcount;
1726 }
1727 if (m != NULL) {
1728 m_freem_list(m);
1729 }
1730 d->bd_hbuf_write = false;
1731 wakeup((caddr_t)d);
1732 bpf_release_d(d);
1733 lck_mtx_unlock(bpf_mlock);
1734
1735 return error;
1736 }
1737
1738 /*
1739 * Reset a descriptor by flushing its packet buffer and clearing the
1740 * receive and drop counts.
1741 */
1742 static void
reset_d(struct bpf_d * d)1743 reset_d(struct bpf_d *d)
1744 {
1745 if (d->bd_hbuf_read) {
1746 panic("resetting buffers during read");
1747 }
1748
1749 if (d->bd_hbuf) {
1750 /* Free the hold buffer. */
1751 d->bd_fbuf = d->bd_hbuf;
1752 d->bd_hbuf = NULL;
1753 }
1754 d->bd_slen = 0;
1755 d->bd_hlen = 0;
1756 d->bd_scnt = 0;
1757 d->bd_hcnt = 0;
1758 d->bd_rcount = 0;
1759 d->bd_dcount = 0;
1760 d->bd_fcount = 0;
1761 d->bd_wcount = 0;
1762 d->bd_wdcount = 0;
1763
1764 d->bd_prev_slen = 0;
1765 }
1766
1767 static struct bpf_d *
bpf_get_device_from_uuid(uuid_t uuid)1768 bpf_get_device_from_uuid(uuid_t uuid)
1769 {
1770 unsigned int i;
1771
1772 for (i = 0; i < nbpfilter; i++) {
1773 struct bpf_d *d = bpf_dtab[i];
1774
1775 if (d == NULL || d == BPF_DEV_RESERVED ||
1776 (d->bd_flags & BPF_CLOSING) != 0) {
1777 continue;
1778 }
1779 if (uuid_compare(uuid, d->bd_uuid) == 0) {
1780 return d;
1781 }
1782 }
1783
1784 return NULL;
1785 }
1786
1787 /*
1788 * The BIOCSETUP command "atomically" attach to the interface and
1789 * copy the buffer from another interface. This minimizes the risk
1790 * of missing packet because this is done while holding
1791 * the BPF global lock
1792 */
1793 static int
bpf_setup(struct bpf_d * d_to,uuid_t uuid_from,ifnet_t ifp,struct proc * proc)1794 bpf_setup(struct bpf_d *d_to, uuid_t uuid_from, ifnet_t ifp, struct proc *proc)
1795 {
1796 struct bpf_d *d_from;
1797 int error = 0;
1798
1799 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
1800
1801 /*
1802 * Sanity checks
1803 */
1804 d_from = bpf_get_device_from_uuid(uuid_from);
1805 if (d_from == NULL) {
1806 error = ENOENT;
1807 os_log_error(OS_LOG_DEFAULT,
1808 "%s: uuids not found error %d",
1809 __func__, error);
1810 return error;
1811 }
1812 if (d_from->bd_opened_by != d_to->bd_opened_by) {
1813 error = EACCES;
1814 os_log_error(OS_LOG_DEFAULT,
1815 "%s: processes not matching error %d",
1816 __func__, error);
1817 return error;
1818 }
1819
1820 /*
1821 * Prevent any read or write while copying
1822 */
1823 while (d_to->bd_hbuf_read || d_to->bd_hbuf_write) {
1824 msleep((caddr_t)d_to, bpf_mlock, PRINET, __func__, NULL);
1825 }
1826 d_to->bd_hbuf_read = true;
1827 d_to->bd_hbuf_write = true;
1828
1829 while (d_from->bd_hbuf_read || d_from->bd_hbuf_write) {
1830 msleep((caddr_t)d_from, bpf_mlock, PRINET, __func__, NULL);
1831 }
1832 d_from->bd_hbuf_read = true;
1833 d_from->bd_hbuf_write = true;
1834
1835 /*
1836 * Verify the devices have not been closed
1837 */
1838 if (d_to->bd_flags & BPF_CLOSING) {
1839 error = ENXIO;
1840 os_log_error(OS_LOG_DEFAULT,
1841 "%s: d_to is closing error %d",
1842 __func__, error);
1843 goto done;
1844 }
1845 if (d_from->bd_flags & BPF_CLOSING) {
1846 error = ENXIO;
1847 os_log_error(OS_LOG_DEFAULT,
1848 "%s: d_from is closing error %d",
1849 __func__, error);
1850 goto done;
1851 }
1852
1853 /*
1854 * For now require the same buffer size
1855 */
1856 if (d_from->bd_bufsize != d_to->bd_bufsize) {
1857 error = EINVAL;
1858 os_log_error(OS_LOG_DEFAULT,
1859 "%s: bufsizes not matching error %d",
1860 __func__, error);
1861 goto done;
1862 }
1863
1864 /*
1865 * Copy relevant options and flags
1866 */
1867 d_to->bd_flags = d_from->bd_flags & (BPF_EXTENDED_HDR | BPF_WANT_PKTAP |
1868 BPF_FINALIZE_PKTAP | BPF_TRUNCATE | BPF_PKTHDRV2 |
1869 BPF_COMP_REQ | BPF_COMP_ENABLED);
1870
1871 d_to->bd_headdrop = d_from->bd_headdrop;
1872
1873 /*
1874 * Allocate and copy the buffers
1875 */
1876 error = bpf_allocbufs(d_to);
1877 if (error != 0) {
1878 goto done;
1879 }
1880
1881 /*
1882 * Make sure the buffers are setup as expected by bpf_setif()
1883 */
1884 ASSERT(d_to->bd_hbuf == NULL);
1885 ASSERT(d_to->bd_sbuf != NULL);
1886 ASSERT(d_to->bd_fbuf != NULL);
1887
1888 /*
1889 * Copy the buffers and update the pointers and counts
1890 */
1891 memcpy(d_to->bd_sbuf, d_from->bd_sbuf, d_from->bd_slen);
1892 d_to->bd_slen = d_from->bd_slen;
1893 d_to->bd_scnt = d_from->bd_scnt;
1894
1895 if (d_from->bd_hbuf != NULL) {
1896 d_to->bd_hbuf = d_to->bd_fbuf;
1897 d_to->bd_fbuf = NULL;
1898 memcpy(d_to->bd_hbuf, d_from->bd_hbuf, d_from->bd_hlen);
1899 }
1900 d_to->bd_hlen = d_from->bd_hlen;
1901 d_to->bd_hcnt = d_from->bd_hcnt;
1902
1903 if (d_to->bd_flags & BPF_COMP_REQ) {
1904 ASSERT(d_to->bd_prev_sbuf != NULL);
1905 ASSERT(d_to->bd_prev_fbuf != NULL);
1906
1907 d_to->bd_prev_slen = d_from->bd_prev_slen;
1908 ASSERT(d_to->bd_prev_slen <= BPF_HDR_COMP_LEN_MAX);
1909 memcpy(d_to->bd_prev_sbuf, d_from->bd_prev_sbuf, BPF_HDR_COMP_LEN_MAX);
1910 }
1911
1912 d_to->bd_bcs = d_from->bd_bcs;
1913
1914 /*
1915 * Attach to the interface:
1916 * - don't reset the buffers
1917 * - we already prevent reads and writes
1918 * - the buffers are already allocated
1919 */
1920 error = bpf_setif(d_to, ifp, false, true, true, proc);
1921 if (error != 0) {
1922 os_log_error(OS_LOG_DEFAULT,
1923 "%s: bpf_setif() failed error %d",
1924 __func__, error);
1925 goto done;
1926 }
1927 done:
1928 d_from->bd_hbuf_read = false;
1929 d_from->bd_hbuf_write = false;
1930 wakeup((caddr_t)d_from);
1931
1932 d_to->bd_hbuf_read = false;
1933 d_to->bd_hbuf_write = false;
1934 wakeup((caddr_t)d_to);
1935
1936 return error;
1937 }
1938
1939 #if DEVELOPMENT || DEBUG
1940 #define BPF_IOC_LIST \
1941 X(FIONREAD) \
1942 X(SIOCGIFADDR) \
1943 X(BIOCGBLEN) \
1944 X(BIOCSBLEN) \
1945 X(BIOCSETF32) \
1946 X(BIOCSETFNR32) \
1947 X(BIOCSETF64) \
1948 X(BIOCSETFNR64) \
1949 X(BIOCFLUSH) \
1950 X(BIOCPROMISC) \
1951 X(BIOCGDLT) \
1952 X(BIOCGDLTLIST) \
1953 X(BIOCSDLT) \
1954 X(BIOCGETIF) \
1955 X(BIOCSETIF) \
1956 X(BIOCSRTIMEOUT32) \
1957 X(BIOCSRTIMEOUT64) \
1958 X(BIOCGRTIMEOUT32) \
1959 X(BIOCGRTIMEOUT64) \
1960 X(BIOCGSTATS) \
1961 X(BIOCIMMEDIATE) \
1962 X(BIOCVERSION) \
1963 X(BIOCGHDRCMPLT) \
1964 X(BIOCSHDRCMPLT) \
1965 X(BIOCGSEESENT) \
1966 X(BIOCSSEESENT) \
1967 X(BIOCSETTC) \
1968 X(BIOCGETTC) \
1969 X(FIONBIO) \
1970 X(FIOASYNC) \
1971 X(BIOCSRSIG) \
1972 X(BIOCGRSIG) \
1973 X(BIOCSEXTHDR) \
1974 X(BIOCGIFATTACHCOUNT) \
1975 X(BIOCGWANTPKTAP) \
1976 X(BIOCSWANTPKTAP) \
1977 X(BIOCSHEADDROP) \
1978 X(BIOCGHEADDROP) \
1979 X(BIOCSTRUNCATE) \
1980 X(BIOCGETUUID) \
1981 X(BIOCSETUP) \
1982 X(BIOCSPKTHDRV2) \
1983 X(BIOCGHDRCOMP) \
1984 X(BIOCSHDRCOMP) \
1985 X(BIOCGHDRCOMPSTATS) \
1986 X(BIOCGHDRCOMPON) \
1987 X(BIOCGDIRECTION) \
1988 X(BIOCSDIRECTION) \
1989 X(BIOCSWRITEMAX) \
1990 X(BIOCGWRITEMAX) \
1991 X(BIOCGBATCHWRITE) \
1992 X(BIOCSBATCHWRITE) \
1993 X(BIOCGNOTSTAMP) \
1994 X(BIOCSNOTSTAMP)
1995
1996 static void
log_bpf_ioctl_str(struct bpf_d * d,u_long cmd,int error)1997 log_bpf_ioctl_str(struct bpf_d *d, u_long cmd, int error)
1998 {
1999 const char *p = NULL;
2000 char str[32];
2001
2002 #define X(x) case x: { p = #x ; printf("%s\n", p); break; }
2003 switch (cmd) {
2004 BPF_IOC_LIST
2005 }
2006 #undef X
2007 if (p == NULL) {
2008 snprintf(str, sizeof(str), "0x%08x", (unsigned int)cmd);
2009 p = str;
2010 }
2011 os_log(OS_LOG_DEFAULT, "bpfioctl bpf%u %s error: %d",
2012 d->bd_dev_minor, p, error);
2013 }
2014 #endif /* DEVELOPMENT || DEBUG */
2015
2016 /*
2017 * FIONREAD Check for read packet available.
2018 * SIOCGIFADDR Get interface address - convenient hook to driver.
2019 * BIOCGBLEN Get buffer len [for read()].
2020 * BIOCSETF Set ethernet read filter.
2021 * BIOCFLUSH Flush read packet buffer.
2022 * BIOCPROMISC Put interface into promiscuous mode.
2023 * BIOCGDLT Get link layer type.
2024 * BIOCGETIF Get interface name.
2025 * BIOCSETIF Set interface.
2026 * BIOCSRTIMEOUT Set read timeout.
2027 * BIOCGRTIMEOUT Get read timeout.
2028 * BIOCGSTATS Get packet stats.
2029 * BIOCIMMEDIATE Set immediate mode.
2030 * BIOCVERSION Get filter language version.
2031 * BIOCGHDRCMPLT Get "header already complete" flag
2032 * BIOCSHDRCMPLT Set "header already complete" flag
2033 * BIOCGSEESENT Get "see packets sent" flag
2034 * BIOCSSEESENT Set "see packets sent" flag
2035 * BIOCSETTC Set traffic class.
2036 * BIOCGETTC Get traffic class.
2037 * BIOCSEXTHDR Set "extended header" flag
2038 * BIOCSHEADDROP Drop head of the buffer if user is not reading
2039 * BIOCGHEADDROP Get "head-drop" flag
2040 */
2041 /* ARGSUSED */
2042 int
bpfioctl(dev_t dev,u_long cmd,caddr_t __sized_by (IOCPARM_LEN (cmd))addr,__unused int flags,struct proc * proc)2043 bpfioctl(dev_t dev, u_long cmd, caddr_t __sized_by(IOCPARM_LEN(cmd)) addr,
2044 __unused int flags, struct proc *proc)
2045 {
2046 struct bpf_d *d;
2047 int error = 0;
2048 u_int int_arg;
2049 struct ifreq ifr = {};
2050
2051 lck_mtx_lock(bpf_mlock);
2052 d = bpf_dtab[minor(dev)];
2053 if (d == NULL || d == BPF_DEV_RESERVED ||
2054 (d->bd_flags & BPF_CLOSING) != 0) {
2055 lck_mtx_unlock(bpf_mlock);
2056 return ENXIO;
2057 }
2058
2059 bpf_acquire_d(d);
2060
2061 if (d->bd_state == BPF_WAITING) {
2062 bpf_stop_timer(d);
2063 }
2064 d->bd_state = BPF_IDLE;
2065
2066 switch (cmd) {
2067 default:
2068 error = EINVAL;
2069 break;
2070
2071 /*
2072 * Check for read packet available.
2073 */
2074 case FIONREAD: { /* int */
2075 int n;
2076
2077 n = d->bd_slen;
2078 if (d->bd_hbuf && d->bd_hbuf_read) {
2079 n += d->bd_hlen;
2080 }
2081
2082 bcopy(&n, addr, sizeof(n));
2083 break;
2084 }
2085
2086 case SIOCGIFADDR: { /* struct ifreq */
2087 struct ifnet *ifp;
2088
2089 if (d->bd_bif == 0) {
2090 error = EINVAL;
2091 } else {
2092 ifp = d->bd_bif->bif_ifp;
2093 error = ifnet_ioctl(ifp, 0, cmd, addr);
2094 }
2095 break;
2096 }
2097
2098 /*
2099 * Get buffer len [for read()].
2100 */
2101 case BIOCGBLEN: { /* u_int */
2102 static_assert(sizeof(d->bd_bufsize) == sizeof(u_int));
2103 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
2104 break;
2105 }
2106
2107 /*
2108 * Set buffer length.
2109 */
2110 case BIOCSBLEN: { /* u_int */
2111 u_int size;
2112
2113 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
2114 /*
2115 * Interface already attached, unable to change buffers
2116 */
2117 error = EINVAL;
2118 break;
2119 }
2120 bcopy(addr, &size, sizeof(size));
2121
2122 if (size > BPF_BUFSIZE_CAP) {
2123 d->bd_bufsize = BPF_BUFSIZE_CAP;
2124
2125 os_log_info(OS_LOG_DEFAULT,
2126 "bpf%d BIOCSBLEN capped to %u from %u",
2127 minor(dev), d->bd_bufsize, size);
2128 } else if (size < BPF_MINBUFSIZE) {
2129 d->bd_bufsize = BPF_MINBUFSIZE;
2130
2131 os_log_info(OS_LOG_DEFAULT,
2132 "bpf%d BIOCSBLEN bumped to %u from %u",
2133 minor(dev), d->bd_bufsize, size);
2134 } else {
2135 d->bd_bufsize = size;
2136
2137 os_log_info(OS_LOG_DEFAULT,
2138 "bpf%d BIOCSBLEN %u",
2139 minor(dev), d->bd_bufsize);
2140 }
2141
2142 /* It's a read/write ioctl */
2143 bcopy(&d->bd_bufsize, addr, sizeof(u_int));
2144 break;
2145 }
2146 /*
2147 * Set link layer read filter.
2148 */
2149 case BIOCSETF32:
2150 case BIOCSETFNR32: { /* struct bpf_program32 */
2151 struct bpf_program32 prg32;
2152
2153 bcopy(addr, &prg32, sizeof(prg32));
2154 error = bpf_setf(d, prg32.bf_len,
2155 CAST_USER_ADDR_T(prg32.bf_insns), cmd);
2156 break;
2157 }
2158
2159 case BIOCSETF64:
2160 case BIOCSETFNR64: { /* struct bpf_program64 */
2161 struct bpf_program64 prg64;
2162
2163 bcopy(addr, &prg64, sizeof(prg64));
2164 error = bpf_setf(d, prg64.bf_len, CAST_USER_ADDR_T(prg64.bf_insns), cmd);
2165 break;
2166 }
2167
2168 /*
2169 * Flush read packet buffer.
2170 */
2171 case BIOCFLUSH:
2172 while (d->bd_hbuf_read) {
2173 msleep((caddr_t)d, bpf_mlock, PRINET, "BIOCFLUSH",
2174 NULL);
2175 }
2176 if ((d->bd_flags & BPF_CLOSING) != 0) {
2177 error = ENXIO;
2178 break;
2179 }
2180 reset_d(d);
2181 break;
2182
2183 /*
2184 * Put interface into promiscuous mode.
2185 */
2186 case BIOCPROMISC:
2187 if (d->bd_bif == 0) {
2188 /*
2189 * No interface attached yet.
2190 */
2191 error = EINVAL;
2192 break;
2193 }
2194 if (d->bd_promisc == 0) {
2195 lck_mtx_unlock(bpf_mlock);
2196 error = ifnet_set_promiscuous(d->bd_bif->bif_ifp, 1);
2197 lck_mtx_lock(bpf_mlock);
2198 if (error == 0) {
2199 d->bd_promisc = 1;
2200 }
2201 }
2202 break;
2203
2204 /*
2205 * Get device parameters.
2206 */
2207 case BIOCGDLT: { /* u_int */
2208 if (d->bd_bif == 0) {
2209 error = EINVAL;
2210 } else {
2211 static_assert(sizeof(d->bd_bif->bif_dlt) == sizeof(u_int));
2212 bcopy(&d->bd_bif->bif_dlt, addr, sizeof(u_int));
2213 }
2214 break;
2215 }
2216
2217 /*
2218 * Get a list of supported data link types.
2219 */
2220 case BIOCGDLTLIST: /* struct bpf_dltlist */
2221 if (d->bd_bif == NULL) {
2222 error = EINVAL;
2223 } else {
2224 error = bpf_getdltlist(d, addr, proc);
2225 }
2226 break;
2227
2228 /*
2229 * Set data link type.
2230 */
2231 case BIOCSDLT: /* u_int */
2232 if (d->bd_bif == NULL) {
2233 error = EINVAL;
2234 } else {
2235 u_int dlt;
2236
2237 bcopy(addr, &dlt, sizeof(dlt));
2238
2239 if (dlt == DLT_PKTAP &&
2240 !(d->bd_flags & BPF_WANT_PKTAP)) {
2241 dlt = DLT_RAW;
2242 }
2243 error = bpf_setdlt(d, dlt, proc);
2244 }
2245 break;
2246
2247 /*
2248 * Get interface name.
2249 */
2250 case BIOCGETIF: /* struct ifreq */
2251 if (d->bd_bif == 0) {
2252 error = EINVAL;
2253 } else {
2254 struct ifnet *const ifp = d->bd_bif->bif_ifp;
2255
2256 snprintf(((struct ifreq *)(void *)addr)->ifr_name,
2257 sizeof(ifr.ifr_name), "%s", if_name(ifp));
2258 }
2259 break;
2260
2261 /*
2262 * Set interface.
2263 */
2264 case BIOCSETIF: { /* struct ifreq */
2265 ifnet_t ifp;
2266
2267 bcopy(addr, &ifr, sizeof(ifr));
2268 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2269 ifp = ifunit(__unsafe_null_terminated_from_indexable(ifr.ifr_name,
2270 &ifr.ifr_name[IFNAMSIZ - 1]));
2271 if (ifp == NULL) {
2272 error = ENXIO;
2273 } else {
2274 error = bpf_setif(d, ifp, true, false, false, proc);
2275 }
2276 break;
2277 }
2278
2279 /*
2280 * Set read timeout.
2281 */
2282 case BIOCSRTIMEOUT32: { /* struct user32_timeval */
2283 struct user32_timeval _tv;
2284 struct timeval tv;
2285
2286 bcopy(addr, &_tv, sizeof(_tv));
2287 tv.tv_sec = _tv.tv_sec;
2288 tv.tv_usec = _tv.tv_usec;
2289
2290 /*
2291 * Subtract 1 tick from tvtohz() since this isn't
2292 * a one-shot timer.
2293 */
2294 if ((error = itimerfix(&tv)) == 0) {
2295 d->bd_rtout = tvtohz(&tv) - 1;
2296 }
2297 break;
2298 }
2299
2300 case BIOCSRTIMEOUT64: { /* struct user64_timeval */
2301 struct user64_timeval _tv;
2302 struct timeval tv;
2303
2304 bcopy(addr, &_tv, sizeof(_tv));
2305 tv.tv_sec = (__darwin_time_t)_tv.tv_sec;
2306 tv.tv_usec = _tv.tv_usec;
2307
2308 /*
2309 * Subtract 1 tick from tvtohz() since this isn't
2310 * a one-shot timer.
2311 */
2312 if ((error = itimerfix(&tv)) == 0) {
2313 d->bd_rtout = tvtohz(&tv) - 1;
2314 }
2315 break;
2316 }
2317
2318 /*
2319 * Get read timeout.
2320 */
2321 case BIOCGRTIMEOUT32: { /* struct user32_timeval */
2322 struct user32_timeval tv;
2323
2324 bzero(&tv, sizeof(tv));
2325 tv.tv_sec = d->bd_rtout / hz;
2326 tv.tv_usec = (d->bd_rtout % hz) * tick;
2327 bcopy(&tv, addr, sizeof(tv));
2328 break;
2329 }
2330
2331 case BIOCGRTIMEOUT64: { /* struct user64_timeval */
2332 struct user64_timeval tv;
2333
2334 bzero(&tv, sizeof(tv));
2335 tv.tv_sec = d->bd_rtout / hz;
2336 tv.tv_usec = (d->bd_rtout % hz) * tick;
2337 bcopy(&tv, addr, sizeof(tv));
2338 break;
2339 }
2340
2341 /*
2342 * Get packet stats.
2343 */
2344 case BIOCGSTATS: { /* struct bpf_stat */
2345 struct bpf_stat bs;
2346
2347 bzero(&bs, sizeof(bs));
2348 bs.bs_recv = (u_int)d->bd_rcount;
2349 bs.bs_drop = (u_int)d->bd_dcount;
2350 bcopy(&bs, addr, sizeof(bs));
2351 break;
2352 }
2353
2354 /*
2355 * Set immediate mode.
2356 */
2357 case BIOCIMMEDIATE: /* u_int */
2358 d->bd_immediate = *(u_char *)(void *)addr;
2359 break;
2360
2361 case BIOCVERSION: { /* struct bpf_version */
2362 struct bpf_version bv;
2363
2364 bzero(&bv, sizeof(bv));
2365 bv.bv_major = BPF_MAJOR_VERSION;
2366 bv.bv_minor = BPF_MINOR_VERSION;
2367 bcopy(&bv, addr, sizeof(bv));
2368 break;
2369 }
2370
2371 /*
2372 * Get "header already complete" flag
2373 */
2374 case BIOCGHDRCMPLT: { /* u_int */
2375 u_int *addr_int = (u_int *)(void *)addr;
2376 *addr_int = d->bd_hdrcmplt;
2377 break;
2378 }
2379
2380 /*
2381 * Set "header already complete" flag
2382 */
2383 case BIOCSHDRCMPLT: /* u_int */
2384 bcopy(addr, &int_arg, sizeof(int_arg));
2385 if (int_arg == 0 && (d->bd_flags & BPF_BATCH_WRITE)) {
2386 os_log(OS_LOG_DEFAULT,
2387 "bpf%u cannot set BIOCSHDRCMPLT when BIOCSBATCHWRITE is set",
2388 d->bd_dev_minor);
2389 error = EINVAL;
2390 break;
2391 }
2392 d->bd_hdrcmplt = int_arg ? 1 : 0;
2393 break;
2394
2395 /*
2396 * Get "see sent packets" flag
2397 */
2398 case BIOCGSEESENT: { /* u_int */
2399 int_arg = 0;
2400
2401 if (d->bd_direction & BPF_D_OUT) {
2402 int_arg = 1;
2403 }
2404 bcopy(&int_arg, addr, sizeof(u_int));
2405 break;
2406 }
2407 /*
2408 * Set "see sent packets" flag
2409 */
2410 case BIOCSSEESENT: { /* u_int */
2411 bcopy(addr, &int_arg, sizeof(u_int));
2412
2413 if (int_arg == 0) {
2414 d->bd_direction = BPF_D_IN;
2415 } else {
2416 d->bd_direction = BPF_D_INOUT;
2417 }
2418 break;
2419 }
2420 /*
2421 * Get direction of tapped packets that can be seen for reading
2422 */
2423 case BIOCGDIRECTION: { /* u_int */
2424 int_arg = d->bd_direction;
2425
2426 bcopy(&int_arg, addr, sizeof(u_int));
2427 break;
2428 }
2429 /*
2430 * Set direction of tapped packets that can be seen for reading
2431 */
2432 case BIOCSDIRECTION: { /* u_int */
2433 bcopy(addr, &int_arg, sizeof(u_int));
2434
2435 switch (int_arg) {
2436 case BPF_D_NONE:
2437 case BPF_D_IN:
2438 case BPF_D_OUT:
2439 case BPF_D_INOUT:
2440 d->bd_direction = int_arg;
2441 break;
2442 default:
2443 error = EINVAL;
2444 break;
2445 }
2446 break;
2447 }
2448 /*
2449 * Set traffic service class
2450 */
2451 case BIOCSETTC: { /* int */
2452 int tc;
2453
2454 bcopy(addr, &tc, sizeof(int));
2455 if (tc != 0 && (d->bd_flags & BPF_BATCH_WRITE)) {
2456 os_log(OS_LOG_DEFAULT,
2457 "bpf%u cannot set BIOCSETTC when BIOCSBATCHWRITE is set",
2458 d->bd_dev_minor);
2459 error = EINVAL;
2460 break;
2461 }
2462 error = bpf_set_traffic_class(d, tc);
2463 break;
2464 }
2465
2466 /*
2467 * Get traffic service class
2468 */
2469 case BIOCGETTC: { /* int */
2470 static_assert(sizeof(d->bd_traffic_class) == sizeof(int));
2471 bcopy(&d->bd_traffic_class, addr, sizeof(int));
2472 break;
2473 }
2474
2475 case FIONBIO: /* Non-blocking I/O; int */
2476 break;
2477
2478 case FIOASYNC: { /* Send signal on receive packets; int */
2479 static_assert(sizeof(d->bd_async) == sizeof(int));
2480 bcopy(addr, &d->bd_async, sizeof(int));
2481 break;
2482 }
2483
2484 case BIOCSRSIG: { /* Set receive signal; u_int */
2485 u_int sig;
2486
2487 bcopy(addr, &sig, sizeof(u_int));
2488
2489 if (sig >= NSIG) {
2490 error = EINVAL;
2491 } else {
2492 d->bd_sig = sig;
2493 }
2494 break;
2495 }
2496 case BIOCGRSIG: { /* u_int */
2497 static_assert(sizeof(d->bd_sig) == sizeof(u_int));
2498 bcopy(&d->bd_sig, addr, sizeof(u_int));
2499 break;
2500 }
2501
2502 case BIOCSEXTHDR: /* u_int */
2503 bcopy(addr, &int_arg, sizeof(int_arg));
2504 if (int_arg) {
2505 d->bd_flags |= BPF_EXTENDED_HDR;
2506 } else {
2507 d->bd_flags &= ~BPF_EXTENDED_HDR;
2508 }
2509 break;
2510
2511 case BIOCGIFATTACHCOUNT: { /* struct ifreq */
2512 ifnet_t ifp;
2513 struct bpf_if *bp;
2514
2515 bcopy(addr, &ifr, sizeof(ifr));
2516 ifr.ifr_name[IFNAMSIZ - 1] = '\0';
2517 ifp = ifunit(__unsafe_null_terminated_from_indexable(ifr.ifr_name,
2518 &ifr.ifr_name[IFNAMSIZ - 1]));
2519 if (ifp == NULL) {
2520 error = ENXIO;
2521 break;
2522 }
2523 ifr.ifr_intval = 0;
2524 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2525 struct bpf_d *bpf_d;
2526
2527 if (bp->bif_ifp == NULL || bp->bif_ifp != ifp) {
2528 continue;
2529 }
2530 for (bpf_d = bp->bif_dlist; bpf_d;
2531 bpf_d = bpf_d->bd_next) {
2532 ifr.ifr_intval += 1;
2533 }
2534 }
2535 bcopy(&ifr, addr, sizeof(ifr));
2536 break;
2537 }
2538 case BIOCGWANTPKTAP: /* u_int */
2539 int_arg = d->bd_flags & BPF_WANT_PKTAP ? 1 : 0;
2540 bcopy(&int_arg, addr, sizeof(int_arg));
2541 break;
2542
2543 case BIOCSWANTPKTAP: /* u_int */
2544 bcopy(addr, &int_arg, sizeof(int_arg));
2545 if (int_arg) {
2546 d->bd_flags |= BPF_WANT_PKTAP;
2547 } else {
2548 d->bd_flags &= ~BPF_WANT_PKTAP;
2549 }
2550 break;
2551
2552 case BIOCSHEADDROP:
2553 bcopy(addr, &int_arg, sizeof(int_arg));
2554 d->bd_headdrop = int_arg ? 1 : 0;
2555 break;
2556
2557 case BIOCGHEADDROP: {
2558 u_int *addr_int = (uint *)(void *)addr;
2559 *addr_int = d->bd_headdrop;
2560 break;
2561 }
2562
2563 case BIOCSTRUNCATE:
2564 bcopy(addr, &int_arg, sizeof(int_arg));
2565 if (int_arg) {
2566 d->bd_flags |= BPF_TRUNCATE;
2567 } else {
2568 d->bd_flags &= ~BPF_TRUNCATE;
2569 }
2570 break;
2571
2572 case BIOCGETUUID: /* uuid_t */
2573 bcopy(&d->bd_uuid, addr, sizeof(uuid_t));
2574 break;
2575
2576 case BIOCSETUP: {
2577 struct bpf_setup_args bsa;
2578 ifnet_t ifp;
2579
2580 bcopy(addr, &bsa, sizeof(struct bpf_setup_args));
2581 bsa.bsa_ifname[IFNAMSIZ - 1] = 0;
2582 ifp = ifunit(__unsafe_null_terminated_from_indexable(bsa.bsa_ifname,
2583 &bsa.bsa_ifname[IFNAMSIZ - 1]));
2584 if (ifp == NULL) {
2585 error = ENXIO;
2586 os_log_error(OS_LOG_DEFAULT,
2587 "%s: ifnet not found for %s error %d",
2588 __func__, bsa.bsa_ifname, error);
2589 break;
2590 }
2591
2592 error = bpf_setup(d, bsa.bsa_uuid, ifp, proc);
2593 break;
2594 }
2595 case BIOCSPKTHDRV2:
2596 bcopy(addr, &int_arg, sizeof(int_arg));
2597 if (int_arg != 0) {
2598 d->bd_flags |= BPF_PKTHDRV2;
2599 } else {
2600 d->bd_flags &= ~BPF_PKTHDRV2;
2601 }
2602 break;
2603
2604 case BIOCGPKTHDRV2:
2605 int_arg = d->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
2606 bcopy(&int_arg, addr, sizeof(int_arg));
2607 break;
2608
2609 case BIOCGHDRCOMP:
2610 int_arg = d->bd_flags & BPF_COMP_REQ ? 1 : 0;
2611 bcopy(&int_arg, addr, sizeof(int_arg));
2612 break;
2613
2614 case BIOCSHDRCOMP:
2615 bcopy(addr, &int_arg, sizeof(int_arg));
2616 if (int_arg != 0 && int_arg != 1) {
2617 return EINVAL;
2618 }
2619 if (d->bd_bif != 0 || (d->bd_flags & BPF_DETACHING)) {
2620 /*
2621 * Interface already attached, unable to change buffers
2622 */
2623 error = EINVAL;
2624 break;
2625 }
2626 if (int_arg != 0) {
2627 d->bd_flags |= BPF_COMP_REQ;
2628 if (bpf_hdr_comp_enable != 0) {
2629 d->bd_flags |= BPF_COMP_ENABLED;
2630 }
2631 } else {
2632 d->bd_flags &= ~(BPF_COMP_REQ | BPF_COMP_ENABLED);
2633 }
2634 break;
2635
2636 case BIOCGHDRCOMPON:
2637 int_arg = d->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
2638 bcopy(&int_arg, addr, sizeof(int_arg));
2639 break;
2640
2641 case BIOCGHDRCOMPSTATS: {
2642 struct bpf_comp_stats bcs = {};
2643
2644 bcs = d->bd_bcs;
2645
2646 bcopy(&bcs, addr, sizeof(bcs));
2647 break;
2648 }
2649 case BIOCSWRITEMAX:
2650 bcopy(addr, &int_arg, sizeof(int_arg));
2651 if (int_arg > BPF_WRITE_MAX) {
2652 os_log(OS_LOG_DEFAULT, "bpf%u bd_write_size_max %u too big",
2653 d->bd_dev_minor, d->bd_write_size_max);
2654 error = EINVAL;
2655 break;
2656 }
2657 d->bd_write_size_max = int_arg;
2658 break;
2659
2660 case BIOCGWRITEMAX:
2661 int_arg = d->bd_write_size_max;
2662 bcopy(&int_arg, addr, sizeof(int_arg));
2663 break;
2664
2665 case BIOCGBATCHWRITE: /* int */
2666 int_arg = d->bd_flags & BPF_BATCH_WRITE ? 1 : 0;
2667 bcopy(&int_arg, addr, sizeof(int_arg));
2668 break;
2669
2670 case BIOCSBATCHWRITE: /* int */
2671 bcopy(addr, &int_arg, sizeof(int_arg));
2672 if (int_arg != 0) {
2673 if (d->bd_hdrcmplt == 0) {
2674 os_log(OS_LOG_DEFAULT,
2675 "bpf%u cannot set BIOCSBATCHWRITE when BIOCSHDRCMPLT is not set",
2676 d->bd_dev_minor);
2677 error = EINVAL;
2678 break;
2679 }
2680 if (d->bd_traffic_class != 0) {
2681 os_log(OS_LOG_DEFAULT,
2682 "bpf%u cannot set BIOCSBATCHWRITE when BIOCSETTC is set",
2683 d->bd_dev_minor);
2684 error = EINVAL;
2685 break;
2686 }
2687 d->bd_flags |= BPF_BATCH_WRITE;
2688 } else {
2689 d->bd_flags &= ~BPF_BATCH_WRITE;
2690 }
2691 break;
2692 case BIOCGNOTSTAMP:
2693 if (d->bd_tstamp == BPF_T_NONE) {
2694 int_arg = 1;
2695 } else {
2696 int_arg = 0;
2697 }
2698 bcopy(&int_arg, addr, sizeof(int_arg));
2699 break;
2700 case BIOCSNOTSTAMP:
2701 bcopy(addr, &int_arg, sizeof(int_arg));
2702 switch (int_arg) {
2703 case 0:
2704 d->bd_tstamp = BPF_T_MICROTIME;
2705 break;
2706 default:
2707 d->bd_tstamp = BPF_T_NONE;
2708 break;
2709 }
2710 break;
2711 case BIOCGDVRTIN:
2712 int_arg = d->bd_flags & BPF_DIVERT_IN ? 1 : 0;
2713 bcopy(&int_arg, addr, sizeof(int_arg));
2714 break;
2715 case BIOCSDVRTIN:
2716 if (d->bd_bif == NULL) {
2717 /*
2718 * No interface attached yet.
2719 */
2720 error = ENXIO;
2721 break;
2722 }
2723 bcopy(addr, &int_arg, sizeof(int_arg));
2724 if (int_arg == 0) {
2725 if ((d->bd_flags & BPF_DIVERT_IN) == 0) {
2726 error = EINVAL;
2727 break;
2728 }
2729 d->bd_flags &= ~BPF_DIVERT_IN;
2730 if_clear_xflags(d->bd_bif->bif_ifp, IFXF_DISABLE_INPUT);
2731 } else {
2732 if ((d->bd_flags & BPF_DIVERT_IN) != 0 ||
2733 (d->bd_bif->bif_ifp->if_xflags & IFXF_DISABLE_INPUT) != 0) {
2734 error = EALREADY;
2735 break;
2736 }
2737 d->bd_flags |= BPF_DIVERT_IN;
2738 if_set_xflags(d->bd_bif->bif_ifp, IFXF_DISABLE_INPUT);
2739 }
2740 break;
2741 }
2742
2743 #if DEVELOPMENT || DEBUG
2744 if (bpf_debug > 0) {
2745 log_bpf_ioctl_str(d, cmd, error);
2746 }
2747 #endif /* DEVELOPMENT || DEBUG */
2748
2749
2750 bpf_release_d(d);
2751 lck_mtx_unlock(bpf_mlock);
2752
2753 return error;
2754 }
2755
2756 /*
2757 * Set d's packet filter program to fp. If this file already has a filter,
2758 * free it and replace it. Returns EINVAL for bogus requests.
2759 */
2760 static int
bpf_setf(struct bpf_d * d,u_int bf_len,user_addr_t bf_insns,u_long cmd)2761 bpf_setf(struct bpf_d *d, u_int bf_len, user_addr_t bf_insns,
2762 u_long cmd)
2763 {
2764 struct bpf_insn *fcode, *old;
2765 u_int flen, size;
2766
2767 while (d->bd_hbuf_read) {
2768 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setf", NULL);
2769 }
2770
2771 if ((d->bd_flags & BPF_CLOSING) != 0) {
2772 return ENXIO;
2773 }
2774
2775 old = d->bd_filter;
2776 if (bf_insns == USER_ADDR_NULL) {
2777 if (bf_len != 0) {
2778 return EINVAL;
2779 }
2780 d->bd_filter = NULL;
2781 d->bd_filter_len = 0;
2782 reset_d(d);
2783 if (old != 0) {
2784 kfree_data_addr(old);
2785 }
2786 return 0;
2787 }
2788 flen = bf_len;
2789 if (flen > BPF_MAXINSNS) {
2790 return EINVAL;
2791 }
2792
2793 size = flen * sizeof(struct bpf_insn);
2794 fcode = (struct bpf_insn *) kalloc_data(size, Z_WAITOK | Z_ZERO);
2795 if (fcode == NULL) {
2796 return ENOMEM;
2797 }
2798 if (copyin(bf_insns, (caddr_t)fcode, size) == 0 &&
2799 bpf_validate(fcode, (int)flen)) {
2800 d->bd_filter = fcode;
2801 d->bd_filter_len = flen;
2802
2803 if (cmd == BIOCSETF32 || cmd == BIOCSETF64) {
2804 reset_d(d);
2805 }
2806
2807 if (old != 0) {
2808 kfree_data_addr(old);
2809 }
2810
2811 return 0;
2812 }
2813 kfree_data(fcode, size);
2814 return EINVAL;
2815 }
2816
2817 /*
2818 * Detach a file from its current interface (if attached at all) and attach
2819 * to the interface indicated by the name stored in ifr.
2820 * Return an errno or 0.
2821 */
2822 static int
bpf_setif(struct bpf_d * d,ifnet_t theywant,bool do_reset,bool has_hbuf_read_write,bool has_bufs_allocated,struct proc * proc)2823 bpf_setif(struct bpf_d *d, ifnet_t theywant, bool do_reset, bool has_hbuf_read_write,
2824 bool has_bufs_allocated, struct proc *proc)
2825 {
2826 struct bpf_if *bp;
2827 int error;
2828
2829 while (!has_hbuf_read_write && (d->bd_hbuf_read || d->bd_hbuf_write)) {
2830 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setif", NULL);
2831 }
2832
2833 if ((d->bd_flags & BPF_CLOSING) != 0) {
2834 return ENXIO;
2835 }
2836
2837 /*
2838 * Look through attached interfaces for the named one.
2839 */
2840 for (bp = bpf_iflist; bp != 0; bp = bp->bif_next) {
2841 struct ifnet *ifp = bp->bif_ifp;
2842
2843 if (ifp == 0 || ifp != theywant) {
2844 continue;
2845 }
2846 /*
2847 * Do not use DLT_PKTAP, unless requested explicitly
2848 */
2849 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2850 continue;
2851 }
2852 /*
2853 * Skip the coprocessor interface
2854 */
2855 if (!intcoproc_unrestricted && IFNET_IS_INTCOPROC(ifp)) {
2856 continue;
2857 }
2858 /*
2859 * We found the requested interface.
2860 * Allocate the packet buffers.
2861 */
2862 if (has_bufs_allocated == false) {
2863 error = bpf_allocbufs(d);
2864 if (error != 0) {
2865 return error;
2866 }
2867 }
2868 /*
2869 * Detach if attached to something else.
2870 */
2871 if (bp != d->bd_bif) {
2872 if (d->bd_bif != NULL) {
2873 if (bpf_detachd(d, proc) != 0) {
2874 return ENXIO;
2875 }
2876 }
2877 if (bpf_attachd(d, bp) != 0) {
2878 return ENXIO;
2879 }
2880 }
2881 if (do_reset) {
2882 reset_d(d);
2883 }
2884 os_log(OS_LOG_DEFAULT, "bpf%u attached to %s by %s:%u",
2885 d->bd_dev_minor, if_name(theywant), proc_name_address(proc), proc_pid(proc));
2886 return 0;
2887 }
2888 /* Not found. */
2889 return ENXIO;
2890 }
2891
2892 /*
2893 * Get a list of available data link type of the interface.
2894 */
2895 static int
bpf_getdltlist(struct bpf_d * d,caddr_t __bidi_indexable addr,struct proc * proc)2896 bpf_getdltlist(struct bpf_d *d, caddr_t __bidi_indexable addr, struct proc *proc)
2897 {
2898 u_int n;
2899 int error;
2900 struct ifnet *ifp;
2901 struct bpf_if *bp;
2902 user_addr_t dlist;
2903 struct bpf_dltlist bfl;
2904
2905 bcopy(addr, &bfl, sizeof(bfl));
2906 if (proc_is64bit(proc)) {
2907 dlist = (user_addr_t)bfl.bfl_u.bflu_pad;
2908 } else {
2909 dlist = CAST_USER_ADDR_T(bfl.bfl_u.bflu_list);
2910 }
2911
2912 ifp = d->bd_bif->bif_ifp;
2913 n = 0;
2914 error = 0;
2915
2916 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2917 if (bp->bif_ifp != ifp) {
2918 continue;
2919 }
2920 /*
2921 * Do not use DLT_PKTAP, unless requested explicitly
2922 */
2923 if (bp->bif_dlt == DLT_PKTAP && !(d->bd_flags & BPF_WANT_PKTAP)) {
2924 continue;
2925 }
2926 if (dlist != USER_ADDR_NULL) {
2927 if (n >= bfl.bfl_len) {
2928 return ENOMEM;
2929 }
2930 error = copyout(&bp->bif_dlt, dlist,
2931 sizeof(bp->bif_dlt));
2932 if (error != 0) {
2933 break;
2934 }
2935 dlist += sizeof(bp->bif_dlt);
2936 }
2937 n++;
2938 }
2939 bfl.bfl_len = n;
2940 bcopy(&bfl, addr, sizeof(bfl));
2941
2942 return error;
2943 }
2944
2945 /*
2946 * Set the data link type of a BPF instance.
2947 */
2948 static int
bpf_setdlt(struct bpf_d * d,uint32_t dlt,struct proc * proc)2949 bpf_setdlt(struct bpf_d *d, uint32_t dlt, struct proc *proc)
2950 {
2951 int error, opromisc;
2952 struct ifnet *ifp;
2953 struct bpf_if *bp;
2954
2955 if (d->bd_bif->bif_dlt == dlt) {
2956 return 0;
2957 }
2958
2959 while (d->bd_hbuf_read) {
2960 msleep((caddr_t)d, bpf_mlock, PRINET, "bpf_setdlt", NULL);
2961 }
2962
2963 if ((d->bd_flags & BPF_CLOSING) != 0) {
2964 return ENXIO;
2965 }
2966
2967 ifp = d->bd_bif->bif_ifp;
2968 for (bp = bpf_iflist; bp; bp = bp->bif_next) {
2969 if (bp->bif_ifp == ifp && bp->bif_dlt == dlt) {
2970 /*
2971 * Do not use DLT_PKTAP, unless requested explicitly
2972 */
2973 if (bp->bif_dlt == DLT_PKTAP &&
2974 !(d->bd_flags & BPF_WANT_PKTAP)) {
2975 continue;
2976 }
2977 break;
2978 }
2979 }
2980 if (bp != NULL) {
2981 opromisc = d->bd_promisc;
2982 if (bpf_detachd(d, proc) != 0) {
2983 return ENXIO;
2984 }
2985 error = bpf_attachd(d, bp);
2986 if (error != 0) {
2987 os_log_error(OS_LOG_DEFAULT,
2988 "bpf_setdlt: bpf%d bpf_attachd %s error %d",
2989 d->bd_dev_minor, if_name(bp->bif_ifp),
2990 error);
2991 return error;
2992 }
2993 reset_d(d);
2994 if (opromisc) {
2995 lck_mtx_unlock(bpf_mlock);
2996 error = ifnet_set_promiscuous(bp->bif_ifp, 1);
2997 lck_mtx_lock(bpf_mlock);
2998 if (error != 0) {
2999 os_log_error(OS_LOG_DEFAULT,
3000 "bpf_setdlt: bpf%d ifpromisc %s error %d",
3001 d->bd_dev_minor, if_name(bp->bif_ifp), error);
3002 } else {
3003 d->bd_promisc = 1;
3004 }
3005 }
3006 }
3007 return bp == NULL ? EINVAL : 0;
3008 }
3009
3010 static int
bpf_set_traffic_class(struct bpf_d * d,int tc)3011 bpf_set_traffic_class(struct bpf_d *d, int tc)
3012 {
3013 int error = 0;
3014
3015 if (!SO_VALID_TC(tc)) {
3016 error = EINVAL;
3017 } else {
3018 d->bd_traffic_class = tc;
3019 }
3020
3021 return error;
3022 }
3023
3024 static void
bpf_set_packet_service_class(struct mbuf * m,int tc)3025 bpf_set_packet_service_class(struct mbuf *m, int tc)
3026 {
3027 if (!(m->m_flags & M_PKTHDR)) {
3028 return;
3029 }
3030
3031 VERIFY(SO_VALID_TC(tc));
3032 (void) m_set_service_class(m, so_tc2msc(tc));
3033 }
3034
3035 /*
3036 * Support for select()
3037 *
3038 * Return true iff the specific operation will not block indefinitely.
3039 * Otherwise, return false but make a note that a selwakeup() must be done.
3040 */
3041 int
bpfselect(dev_t dev,int which,void * wql,struct proc * proc)3042 bpfselect(dev_t dev, int which, void * wql, struct proc *proc)
3043 {
3044 struct bpf_d *d;
3045 int ret = 0;
3046
3047 lck_mtx_lock(bpf_mlock);
3048
3049 d = bpf_dtab[minor(dev)];
3050 if (d == NULL || d == BPF_DEV_RESERVED ||
3051 (d->bd_flags & BPF_CLOSING) != 0) {
3052 lck_mtx_unlock(bpf_mlock);
3053 return ENXIO;
3054 }
3055
3056 bpf_acquire_d(d);
3057
3058 if (d->bd_bif == NULL) {
3059 bpf_release_d(d);
3060 lck_mtx_unlock(bpf_mlock);
3061 return ENXIO;
3062 }
3063
3064 while (d->bd_hbuf_read) {
3065 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfselect", NULL);
3066 }
3067
3068 if ((d->bd_flags & BPF_CLOSING) != 0) {
3069 bpf_release_d(d);
3070 lck_mtx_unlock(bpf_mlock);
3071 return ENXIO;
3072 }
3073
3074 switch (which) {
3075 case FREAD:
3076 if (d->bd_hlen != 0 ||
3077 ((d->bd_immediate ||
3078 d->bd_state == BPF_TIMED_OUT) && d->bd_slen != 0)) {
3079 ret = 1; /* read has data to return */
3080 } else {
3081 /*
3082 * Read has no data to return.
3083 * Make the select wait, and start a timer if
3084 * necessary.
3085 */
3086 selrecord(proc, &d->bd_sel, wql);
3087 bpf_start_timer(d);
3088 }
3089 break;
3090
3091 case FWRITE:
3092 /* can't determine whether a write would block */
3093 ret = 1;
3094 break;
3095 }
3096
3097 bpf_release_d(d);
3098 lck_mtx_unlock(bpf_mlock);
3099
3100 return ret;
3101 }
3102
3103 /*
3104 * Support for kevent() system call. Register EVFILT_READ filters and
3105 * reject all others.
3106 */
3107 int bpfkqfilter(dev_t dev, struct knote *kn);
3108 static void filt_bpfdetach(struct knote *);
3109 static int filt_bpfread(struct knote *, long);
3110 static int filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev);
3111 static int filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev);
3112
3113 SECURITY_READ_ONLY_EARLY(struct filterops) bpfread_filtops = {
3114 .f_isfd = 1,
3115 .f_detach = filt_bpfdetach,
3116 .f_event = filt_bpfread,
3117 .f_touch = filt_bpftouch,
3118 .f_process = filt_bpfprocess,
3119 };
3120
3121 static int
filt_bpfread_common(struct knote * kn,struct kevent_qos_s * kev,struct bpf_d * d)3122 filt_bpfread_common(struct knote *kn, struct kevent_qos_s *kev, struct bpf_d *d)
3123 {
3124 int ready = 0;
3125 int64_t data = 0;
3126
3127 if (d->bd_immediate) {
3128 /*
3129 * If there's data in the hold buffer, it's the
3130 * amount of data a read will return.
3131 *
3132 * If there's no data in the hold buffer, but
3133 * there's data in the store buffer, a read will
3134 * immediately rotate the store buffer to the
3135 * hold buffer, the amount of data in the store
3136 * buffer is the amount of data a read will
3137 * return.
3138 *
3139 * If there's no data in either buffer, we're not
3140 * ready to read.
3141 */
3142 data = (d->bd_hlen == 0 || d->bd_hbuf_read ?
3143 d->bd_slen : d->bd_hlen);
3144 int64_t lowwat = knote_low_watermark(kn);
3145 if (lowwat > d->bd_bufsize) {
3146 lowwat = d->bd_bufsize;
3147 }
3148 ready = (data >= lowwat);
3149 } else {
3150 /*
3151 * If there's data in the hold buffer, it's the
3152 * amount of data a read will return.
3153 *
3154 * If there's no data in the hold buffer, but
3155 * there's data in the store buffer, if the
3156 * timer has expired a read will immediately
3157 * rotate the store buffer to the hold buffer,
3158 * so the amount of data in the store buffer is
3159 * the amount of data a read will return.
3160 *
3161 * If there's no data in either buffer, or there's
3162 * no data in the hold buffer and the timer hasn't
3163 * expired, we're not ready to read.
3164 */
3165 data = ((d->bd_hlen == 0 || d->bd_hbuf_read) &&
3166 d->bd_state == BPF_TIMED_OUT ? d->bd_slen : d->bd_hlen);
3167 ready = (data > 0);
3168 }
3169 if (!ready) {
3170 bpf_start_timer(d);
3171 } else if (kev) {
3172 knote_fill_kevent(kn, kev, data);
3173 }
3174
3175 return ready;
3176 }
3177
3178 int
bpfkqfilter(dev_t dev,struct knote * kn)3179 bpfkqfilter(dev_t dev, struct knote *kn)
3180 {
3181 struct bpf_d *d;
3182 int res;
3183
3184 /*
3185 * Is this device a bpf?
3186 */
3187 if (major(dev) != CDEV_MAJOR || kn->kn_filter != EVFILT_READ) {
3188 knote_set_error(kn, EINVAL);
3189 return 0;
3190 }
3191
3192 lck_mtx_lock(bpf_mlock);
3193
3194 d = bpf_dtab[minor(dev)];
3195
3196 if (d == NULL || d == BPF_DEV_RESERVED ||
3197 (d->bd_flags & BPF_CLOSING) != 0 ||
3198 d->bd_bif == NULL) {
3199 lck_mtx_unlock(bpf_mlock);
3200 knote_set_error(kn, ENXIO);
3201 return 0;
3202 }
3203
3204 kn->kn_filtid = EVFILTID_BPFREAD;
3205 knote_kn_hook_set_raw(kn, d);
3206 KNOTE_ATTACH(&d->bd_sel.si_note, kn);
3207 d->bd_flags |= BPF_KNOTE;
3208
3209 /* capture the current state */
3210 res = filt_bpfread_common(kn, NULL, d);
3211
3212 lck_mtx_unlock(bpf_mlock);
3213
3214 return res;
3215 }
3216
3217 static void
filt_bpfdetach(struct knote * kn)3218 filt_bpfdetach(struct knote *kn)
3219 {
3220 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3221
3222 lck_mtx_lock(bpf_mlock);
3223 if (d->bd_flags & BPF_KNOTE) {
3224 KNOTE_DETACH(&d->bd_sel.si_note, kn);
3225 d->bd_flags &= ~BPF_KNOTE;
3226 }
3227 lck_mtx_unlock(bpf_mlock);
3228 }
3229
3230 static int
filt_bpfread(struct knote * kn,long hint)3231 filt_bpfread(struct knote *kn, long hint)
3232 {
3233 #pragma unused(hint)
3234 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3235
3236 return filt_bpfread_common(kn, NULL, d);
3237 }
3238
3239 static int
filt_bpftouch(struct knote * kn,struct kevent_qos_s * kev)3240 filt_bpftouch(struct knote *kn, struct kevent_qos_s *kev)
3241 {
3242 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3243 int res;
3244
3245 lck_mtx_lock(bpf_mlock);
3246
3247 /* save off the lowat threshold and flag */
3248 kn->kn_sdata = kev->data;
3249 kn->kn_sfflags = kev->fflags;
3250
3251 /* output data will be re-generated here */
3252 res = filt_bpfread_common(kn, NULL, d);
3253
3254 lck_mtx_unlock(bpf_mlock);
3255
3256 return res;
3257 }
3258
3259 static int
filt_bpfprocess(struct knote * kn,struct kevent_qos_s * kev)3260 filt_bpfprocess(struct knote *kn, struct kevent_qos_s *kev)
3261 {
3262 struct bpf_d *d = (struct bpf_d *)knote_kn_hook_get_raw(kn);
3263 int res;
3264
3265 lck_mtx_lock(bpf_mlock);
3266 res = filt_bpfread_common(kn, kev, d);
3267 lck_mtx_unlock(bpf_mlock);
3268
3269 return res;
3270 }
3271
3272 /*
3273 * Copy data from an mbuf chain into a buffer. This code is derived
3274 * from m_copydata in kern/uipc_mbuf.c.
3275 */
3276 static void
bpf_mcopy(struct mbuf * m,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3277 bpf_mcopy(struct mbuf *m, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3278 {
3279 u_int count;
3280
3281 while (offset >= m->m_len) {
3282 offset -= m->m_len;
3283 m = m->m_next;
3284 if (m == NULL) {
3285 panic("bpf_mcopy");
3286 }
3287 continue;
3288 }
3289
3290 while (len > 0) {
3291 if (m == NULL) {
3292 panic("bpf_mcopy");
3293 }
3294 count = MIN(m->m_len - (u_int)offset, (u_int)len);
3295 bcopy(m_mtod_current(m) + offset, dst, count);
3296 m = m->m_next;
3297 dst += count;
3298 len -= count;
3299 offset = 0;
3300 }
3301 }
3302
3303 static inline void
bpf_tap_imp(ifnet_t ifp,u_int32_t dlt,struct bpf_packet * bpf_pkt,int outbound)3304 bpf_tap_imp(
3305 ifnet_t ifp,
3306 u_int32_t dlt,
3307 struct bpf_packet *bpf_pkt,
3308 int outbound)
3309 {
3310 struct bpf_d *d;
3311 u_int slen;
3312 struct bpf_if *bp;
3313
3314 /*
3315 * It's possible that we get here after the bpf descriptor has been
3316 * detached from the interface; in such a case we simply return.
3317 * Lock ordering is important since we can be called asynchronously
3318 * (from IOKit) to process an inbound packet; when that happens
3319 * we would have been holding its "gateLock" and will be acquiring
3320 * "bpf_mlock" upon entering this routine. Due to that, we release
3321 * "bpf_mlock" prior to calling ifnet_set_promiscuous (which will
3322 * acquire "gateLock" in the IOKit), in order to avoid a deadlock
3323 * when a ifnet_set_promiscuous request simultaneously collides with
3324 * an inbound packet being passed into the tap callback.
3325 */
3326 lck_mtx_lock(bpf_mlock);
3327 if (ifp->if_bpf == NULL) {
3328 lck_mtx_unlock(bpf_mlock);
3329 return;
3330 }
3331 for (bp = ifp->if_bpf; bp != NULL; bp = bp->bif_next) {
3332 if (bp->bif_ifp != ifp) {
3333 /* wrong interface */
3334 bp = NULL;
3335 break;
3336 }
3337 if (dlt == 0 || bp->bif_dlt == dlt) {
3338 /* tapping default DLT or DLT matches */
3339 break;
3340 }
3341 }
3342 if (bp == NULL) {
3343 goto done;
3344 }
3345 for (d = bp->bif_dlist; d != NULL; d = d->bd_next) {
3346 struct bpf_packet *bpf_pkt_saved = bpf_pkt;
3347 u_char *bpf_pkt_ptr = (u_char *)(struct bpf_packet *__bidi_indexable)bpf_pkt;
3348 struct bpf_packet bpf_pkt_tmp = {};
3349 struct pktap_header_buffer bpfp_header_tmp = {};
3350
3351 if (outbound && (d->bd_direction & BPF_D_OUT) == 0) {
3352 continue;
3353 }
3354 if (!outbound && (d->bd_direction & BPF_D_IN) == 0) {
3355 continue;
3356 }
3357
3358 ++d->bd_rcount;
3359 slen = bpf_filter(d->bd_filter, d->bd_filter_len,
3360 bpf_pkt_ptr,
3361 (u_int)bpf_pkt->bpfp_total_length, 0);
3362
3363 if (slen != 0) {
3364 if (bp->bif_ifp->if_type == IFT_PKTAP &&
3365 bp->bif_dlt == DLT_PKTAP) {
3366 if (d->bd_flags & BPF_TRUNCATE) {
3367 slen = min(slen, get_pkt_trunc_len(bpf_pkt));
3368 }
3369 /*
3370 * Need to copy the bpf_pkt because the conversion
3371 * to v2 pktap header modifies the content of the
3372 * bpfp_header
3373 */
3374 if ((d->bd_flags & BPF_PKTHDRV2) &&
3375 bpf_pkt->bpfp_header_length <= sizeof(bpfp_header_tmp)) {
3376 bpf_pkt_tmp = *bpf_pkt;
3377 bpf_pkt = &bpf_pkt_tmp;
3378
3379 memcpy(&bpfp_header_tmp, bpf_pkt->bpfp_header,
3380 bpf_pkt->bpfp_header_length);
3381
3382 bpf_pkt->bpfp_header = &bpfp_header_tmp;
3383 bpf_pkt->bpfp_header_length = bpf_pkt->bpfp_header_length;
3384
3385 convert_to_pktap_header_to_v2(bpf_pkt,
3386 !!(d->bd_flags & BPF_TRUNCATE));
3387 }
3388 }
3389 ++d->bd_fcount;
3390 catchpacket(d, bpf_pkt, slen, outbound);
3391 }
3392 bpf_pkt = bpf_pkt_saved;
3393 }
3394
3395 done:
3396 lck_mtx_unlock(bpf_mlock);
3397 }
3398
3399 static inline void
bpf_tap_mbuf(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen,int outbound)3400 bpf_tap_mbuf(
3401 ifnet_t ifp,
3402 u_int32_t dlt,
3403 mbuf_t m,
3404 void *__sized_by(hlen) hdr,
3405 size_t hlen,
3406 int outbound)
3407 {
3408 struct bpf_packet bpf_pkt;
3409 mbuf_ref_t m0;
3410
3411 if (ifp->if_bpf == NULL) {
3412 /* quickly check without taking lock */
3413 return;
3414 }
3415 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3416 bpf_pkt.bpfp_mbuf = m;
3417 bpf_pkt.bpfp_total_length = 0;
3418 for (m0 = m; m0 != NULL; m0 = m0->m_next) {
3419 bpf_pkt.bpfp_total_length += m0->m_len;
3420 }
3421 if (hdr != NULL) {
3422 bpf_pkt.bpfp_header = hdr;
3423 bpf_pkt.bpfp_header_length = hlen;
3424 bpf_pkt.bpfp_total_length += hlen;
3425 } else {
3426 bpf_pkt.bpfp_header = NULL;
3427 bpf_pkt.bpfp_header_length = 0;
3428 }
3429 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3430 }
3431
3432 void
bpf_tap_out(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen)3433 bpf_tap_out(
3434 ifnet_t ifp,
3435 u_int32_t dlt,
3436 mbuf_t m,
3437 void *__sized_by(hlen) hdr,
3438 size_t hlen)
3439 {
3440 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 1);
3441 }
3442
3443 void
bpf_tap_in(ifnet_t ifp,u_int32_t dlt,mbuf_t m,void * __sized_by (hlen)hdr,size_t hlen)3444 bpf_tap_in(
3445 ifnet_t ifp,
3446 u_int32_t dlt,
3447 mbuf_t m,
3448 void *__sized_by(hlen) hdr,
3449 size_t hlen)
3450 {
3451 bpf_tap_mbuf(ifp, dlt, m, hdr, hlen, 0);
3452 }
3453
3454 /* Callback registered with Ethernet driver. */
3455 static int
bpf_tap_callback(struct ifnet * ifp,struct mbuf * m)3456 bpf_tap_callback(struct ifnet *ifp, struct mbuf *m)
3457 {
3458 bpf_tap_mbuf(ifp, 0, m, NULL, 0, mbuf_pkthdr_rcvif(m) == NULL);
3459
3460 return 0;
3461 }
3462
3463 #if SKYWALK
3464 #include <skywalk/os_skywalk_private.h>
3465
3466 static void
bpf_pktcopy(kern_packet_t pkt,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3467 bpf_pktcopy(kern_packet_t pkt, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3468 {
3469 kern_buflet_t buflet = NULL;
3470 size_t count;
3471
3472 while (len > 0) {
3473 uint8_t *addr;
3474 uint32_t buflet_length;
3475 uint32_t buflet_offset;
3476 uint32_t limit;
3477
3478 buflet = kern_packet_get_next_buflet(pkt, buflet);
3479 VERIFY(buflet != NULL);
3480 limit = kern_buflet_get_data_limit(buflet);
3481 buflet_offset = kern_buflet_get_data_offset(buflet);
3482 addr = __unsafe_forge_bidi_indexable(uint8_t *,
3483 (uintptr_t)kern_buflet_get_data_address(buflet) + buflet_offset,
3484 limit - buflet_offset);
3485 VERIFY(addr != NULL);
3486 buflet_length = kern_buflet_get_data_length(buflet);
3487 if (offset >= buflet_length) {
3488 offset -= buflet_length;
3489 continue;
3490 }
3491 count = MIN(buflet_length - offset, len);
3492 bcopy((void *)(addr + offset), (void *)dst, count);
3493 dst += count;
3494 len -= count;
3495 offset = 0;
3496 }
3497 }
3498
3499 static inline void
bpf_tap_packet(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen,int outbound)3500 bpf_tap_packet(
3501 ifnet_t ifp,
3502 u_int32_t dlt,
3503 kern_packet_t pkt,
3504 void *__sized_by(hlen) hdr,
3505 size_t hlen,
3506 int outbound)
3507 {
3508 struct bpf_packet bpf_pkt;
3509 struct mbuf * m;
3510
3511 if (ifp->if_bpf == NULL) {
3512 /* quickly check without taking lock */
3513 return;
3514 }
3515 m = kern_packet_get_mbuf(pkt);
3516 if (m != NULL) {
3517 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_MBUF;
3518 bpf_pkt.bpfp_mbuf = m;
3519 bpf_pkt.bpfp_total_length = m_length(m);
3520 } else {
3521 bpf_pkt.bpfp_type = BPF_PACKET_TYPE_PKT;
3522 bpf_pkt.bpfp_pkt = pkt;
3523 if (PKT_ADDR(pkt)->pkt_pflags & PKT_F_TRUNCATED) {
3524 struct __kern_buflet *bft = kern_packet_get_next_buflet(pkt, NULL);
3525 bpf_pkt.bpfp_total_length = kern_buflet_get_data_length(bft);
3526 } else {
3527 bpf_pkt.bpfp_total_length = kern_packet_get_data_length(pkt);
3528 }
3529 }
3530 bpf_pkt.bpfp_header = hdr;
3531 bpf_pkt.bpfp_header_length = hlen;
3532 if (hlen != 0) {
3533 bpf_pkt.bpfp_total_length += hlen;
3534 }
3535 bpf_tap_imp(ifp, dlt, &bpf_pkt, outbound);
3536 }
3537
3538 void
bpf_tap_packet_out(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen)3539 bpf_tap_packet_out(
3540 ifnet_t ifp,
3541 u_int32_t dlt,
3542 kern_packet_t pkt,
3543 void *__sized_by(hlen) hdr,
3544 size_t hlen)
3545 {
3546 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 1);
3547 }
3548
3549 void
bpf_tap_packet_in(ifnet_t ifp,u_int32_t dlt,kern_packet_t pkt,void * __sized_by (hlen)hdr,size_t hlen)3550 bpf_tap_packet_in(
3551 ifnet_t ifp,
3552 u_int32_t dlt,
3553 kern_packet_t pkt,
3554 void *__sized_by(hlen) hdr,
3555 size_t hlen)
3556 {
3557 bpf_tap_packet(ifp, dlt, pkt, hdr, hlen, 0);
3558 }
3559
3560 #endif /* SKYWALK */
3561
3562 static errno_t
bpf_copydata(struct bpf_packet * pkt,size_t off,size_t len,void * __sized_by (len)out_data)3563 bpf_copydata(struct bpf_packet *pkt, size_t off, size_t len, void *__sized_by(len) out_data)
3564 {
3565 errno_t err = 0;
3566 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
3567 err = mbuf_copydata(pkt->bpfp_mbuf, off, len, out_data);
3568 #if SKYWALK
3569 } else if (pkt->bpfp_type == BPF_PACKET_TYPE_PKT) {
3570 err = kern_packet_copy_bytes(pkt->bpfp_pkt, off, len, out_data);
3571 #endif /* SKYWALK */
3572 } else {
3573 err = EINVAL;
3574 }
3575
3576 return err;
3577 }
3578
3579 static void
copy_bpf_packet_offset(struct bpf_packet * pkt,uint8_t * __sized_by (len)dst,size_t len,size_t offset)3580 copy_bpf_packet_offset(struct bpf_packet * pkt, uint8_t *__sized_by(len) dst, size_t len, size_t offset)
3581 {
3582 /* copy the optional header */
3583 if (offset < pkt->bpfp_header_length) {
3584 size_t count = MIN(len, pkt->bpfp_header_length - offset);
3585 caddr_t src = pkt->bpfp_header;
3586 bcopy(src + offset, dst, count);
3587 dst += count;
3588 len -= count;
3589 offset = 0;
3590 } else {
3591 offset -= pkt->bpfp_header_length;
3592 }
3593
3594 if (len == 0) {
3595 /* nothing past the header */
3596 return;
3597 }
3598 /* copy the packet */
3599 switch (pkt->bpfp_type) {
3600 case BPF_PACKET_TYPE_MBUF:
3601 bpf_mcopy(pkt->bpfp_mbuf, dst, len, offset);
3602 break;
3603 #if SKYWALK
3604 case BPF_PACKET_TYPE_PKT:
3605 bpf_pktcopy(pkt->bpfp_pkt, dst, len, offset);
3606 break;
3607 #endif /* SKYWALK */
3608 default:
3609 break;
3610 }
3611 }
3612
3613 static void
copy_bpf_packet(struct bpf_packet * pkt,uint8_t * __sized_by (len)dst,size_t len)3614 copy_bpf_packet(struct bpf_packet * pkt, uint8_t *__sized_by(len) dst, size_t len)
3615 {
3616 copy_bpf_packet_offset(pkt, dst, len, 0);
3617 }
3618
3619 static uint32_t
get_esp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3620 get_esp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3621 const uint32_t remaining_caplen)
3622 {
3623 /*
3624 * For some reason tcpdump expects to have one byte beyond the ESP header
3625 */
3626 uint32_t trunc_len = ESP_HDR_SIZE + 1;
3627
3628 if (trunc_len > remaining_caplen) {
3629 return remaining_caplen;
3630 }
3631
3632 return trunc_len;
3633 }
3634
3635 static uint32_t
get_isakmp_trunc_len(__unused struct bpf_packet * pkt,__unused uint32_t off,const uint32_t remaining_caplen)3636 get_isakmp_trunc_len(__unused struct bpf_packet *pkt, __unused uint32_t off,
3637 const uint32_t remaining_caplen)
3638 {
3639 /*
3640 * Include the payload generic header
3641 */
3642 uint32_t trunc_len = ISAKMP_HDR_SIZE;
3643
3644 if (trunc_len > remaining_caplen) {
3645 return remaining_caplen;
3646 }
3647
3648 return trunc_len;
3649 }
3650
3651 static uint32_t
get_isakmp_natt_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3652 get_isakmp_natt_trunc_len(struct bpf_packet *pkt, uint32_t off,
3653 const uint32_t remaining_caplen)
3654 {
3655 int err = 0;
3656 uint32_t trunc_len = 0;
3657 char payload[remaining_caplen];
3658
3659 err = bpf_copydata(pkt, off, remaining_caplen, payload);
3660 if (err != 0) {
3661 return remaining_caplen;
3662 }
3663 /*
3664 * They are three cases:
3665 * - IKE: payload start with 4 bytes header set to zero before ISAKMP header
3666 * - keep alive: 1 byte payload
3667 * - otherwise it's ESP
3668 */
3669 if (remaining_caplen >= 4 &&
3670 payload[0] == 0 && payload[1] == 0 &&
3671 payload[2] == 0 && payload[3] == 0) {
3672 trunc_len = 4 + get_isakmp_trunc_len(pkt, off + 4, remaining_caplen - 4);
3673 } else if (remaining_caplen == 1) {
3674 trunc_len = 1;
3675 } else {
3676 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3677 }
3678
3679 if (trunc_len > remaining_caplen) {
3680 return remaining_caplen;
3681 }
3682
3683 return trunc_len;
3684 }
3685
3686 static uint32_t
get_udp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3687 get_udp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3688 {
3689 int err = 0;
3690 uint32_t trunc_len = sizeof(struct udphdr); /* By default no UDP payload */
3691
3692 if (trunc_len >= remaining_caplen) {
3693 return remaining_caplen;
3694 }
3695
3696 struct udphdr udphdr;
3697 err = bpf_copydata(pkt, off, sizeof(struct udphdr), &udphdr);
3698 if (err != 0) {
3699 return remaining_caplen;
3700 }
3701
3702 u_short sport, dport;
3703
3704 sport = EXTRACT_SHORT(&udphdr.uh_sport);
3705 dport = EXTRACT_SHORT(&udphdr.uh_dport);
3706
3707 if (dport == PORT_DNS || sport == PORT_DNS) {
3708 /*
3709 * Full UDP payload for DNS
3710 */
3711 trunc_len = remaining_caplen;
3712 } else if ((sport == PORT_BOOTPS && dport == PORT_BOOTPC) ||
3713 (sport == PORT_BOOTPC && dport == PORT_BOOTPS)) {
3714 /*
3715 * Full UDP payload for BOOTP and DHCP
3716 */
3717 trunc_len = remaining_caplen;
3718 } else if (dport == PORT_ISAKMP && sport == PORT_ISAKMP) {
3719 /*
3720 * Return the ISAKMP header
3721 */
3722 trunc_len += get_isakmp_trunc_len(pkt, off + sizeof(struct udphdr),
3723 remaining_caplen - sizeof(struct udphdr));
3724 } else if (dport == PORT_ISAKMP_NATT && sport == PORT_ISAKMP_NATT) {
3725 trunc_len += get_isakmp_natt_trunc_len(pkt, off + sizeof(struct udphdr),
3726 remaining_caplen - sizeof(struct udphdr));
3727 }
3728 if (trunc_len >= remaining_caplen) {
3729 return remaining_caplen;
3730 }
3731
3732 return trunc_len;
3733 }
3734
3735 static uint32_t
get_tcp_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3736 get_tcp_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3737 {
3738 int err = 0;
3739 uint32_t trunc_len = sizeof(struct tcphdr); /* By default no TCP payload */
3740 if (trunc_len >= remaining_caplen) {
3741 return remaining_caplen;
3742 }
3743
3744 struct tcphdr tcphdr;
3745 err = bpf_copydata(pkt, off, sizeof(struct tcphdr), &tcphdr);
3746 if (err != 0) {
3747 return remaining_caplen;
3748 }
3749
3750 u_short sport, dport;
3751 sport = EXTRACT_SHORT(&tcphdr.th_sport);
3752 dport = EXTRACT_SHORT(&tcphdr.th_dport);
3753
3754 if (dport == PORT_DNS || sport == PORT_DNS) {
3755 /*
3756 * Full TCP payload for DNS
3757 */
3758 trunc_len = remaining_caplen;
3759 } else {
3760 trunc_len = (uint16_t)(tcphdr.th_off << 2);
3761 }
3762 if (trunc_len >= remaining_caplen) {
3763 return remaining_caplen;
3764 }
3765
3766 return trunc_len;
3767 }
3768
3769 static uint32_t
get_proto_trunc_len(uint8_t proto,struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3770 get_proto_trunc_len(uint8_t proto, struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3771 {
3772 uint32_t trunc_len;
3773
3774 switch (proto) {
3775 case IPPROTO_ICMP: {
3776 /*
3777 * Full IMCP payload
3778 */
3779 trunc_len = remaining_caplen;
3780 break;
3781 }
3782 case IPPROTO_ICMPV6: {
3783 /*
3784 * Full IMCPV6 payload
3785 */
3786 trunc_len = remaining_caplen;
3787 break;
3788 }
3789 case IPPROTO_IGMP: {
3790 /*
3791 * Full IGMP payload
3792 */
3793 trunc_len = remaining_caplen;
3794 break;
3795 }
3796 case IPPROTO_UDP: {
3797 trunc_len = get_udp_trunc_len(pkt, off, remaining_caplen);
3798 break;
3799 }
3800 case IPPROTO_TCP: {
3801 trunc_len = get_tcp_trunc_len(pkt, off, remaining_caplen);
3802 break;
3803 }
3804 case IPPROTO_ESP: {
3805 trunc_len = get_esp_trunc_len(pkt, off, remaining_caplen);
3806 break;
3807 }
3808 default: {
3809 /*
3810 * By default we only include the IP header
3811 */
3812 trunc_len = 0;
3813 break;
3814 }
3815 }
3816 if (trunc_len >= remaining_caplen) {
3817 return remaining_caplen;
3818 }
3819
3820 return trunc_len;
3821 }
3822
3823 static uint32_t
get_ip_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3824 get_ip_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3825 {
3826 int err = 0;
3827 uint32_t iplen = sizeof(struct ip);
3828 if (iplen >= remaining_caplen) {
3829 return remaining_caplen;
3830 }
3831
3832 struct ip iphdr;
3833 err = bpf_copydata(pkt, off, sizeof(struct ip), &iphdr);
3834 if (err != 0) {
3835 return remaining_caplen;
3836 }
3837
3838 uint8_t proto = 0;
3839
3840 iplen = (uint16_t)(iphdr.ip_hl << 2);
3841 if (iplen >= remaining_caplen) {
3842 return remaining_caplen;
3843 }
3844
3845 proto = iphdr.ip_p;
3846 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3847
3848 if (iplen >= remaining_caplen) {
3849 return remaining_caplen;
3850 }
3851
3852 return iplen;
3853 }
3854
3855 static uint32_t
get_ip6_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3856 get_ip6_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3857 {
3858 int err = 0;
3859 uint32_t iplen = sizeof(struct ip6_hdr);
3860 if (iplen >= remaining_caplen) {
3861 return remaining_caplen;
3862 }
3863
3864 struct ip6_hdr ip6hdr;
3865 err = bpf_copydata(pkt, off, sizeof(struct ip6_hdr), &ip6hdr);
3866 if (err != 0) {
3867 return remaining_caplen;
3868 }
3869
3870 uint8_t proto = 0;
3871
3872 /*
3873 * TBD: process the extension headers
3874 */
3875 proto = ip6hdr.ip6_nxt;
3876 iplen += get_proto_trunc_len(proto, pkt, off + iplen, remaining_caplen - iplen);
3877
3878 if (iplen >= remaining_caplen) {
3879 return remaining_caplen;
3880 }
3881
3882 return iplen;
3883 }
3884
3885 static uint32_t
get_ether_trunc_len(struct bpf_packet * pkt,uint32_t off,const uint32_t remaining_caplen)3886 get_ether_trunc_len(struct bpf_packet *pkt, uint32_t off, const uint32_t remaining_caplen)
3887 {
3888 int err = 0;
3889 uint32_t ethlen = sizeof(struct ether_header);
3890 if (ethlen >= remaining_caplen) {
3891 return remaining_caplen;
3892 }
3893
3894 struct ether_header eh = {};
3895 err = bpf_copydata(pkt, off, sizeof(struct ether_header), &eh);
3896 if (err != 0) {
3897 return remaining_caplen;
3898 }
3899
3900 u_short type = EXTRACT_SHORT(&eh.ether_type);
3901 /* Include full ARP */
3902 if (type == ETHERTYPE_ARP) {
3903 ethlen = remaining_caplen;
3904 } else if (type == ETHERTYPE_IP) {
3905 ethlen += get_ip_trunc_len(pkt, off + sizeof(struct ether_header),
3906 remaining_caplen - ethlen);
3907 } else if (type == ETHERTYPE_IPV6) {
3908 ethlen += get_ip6_trunc_len(pkt, off + sizeof(struct ether_header),
3909 remaining_caplen - ethlen);
3910 } else {
3911 ethlen = MIN(BPF_MIN_PKT_SIZE, remaining_caplen);
3912 }
3913 return ethlen;
3914 }
3915
3916 static uint32_t
get_pkt_trunc_len(struct bpf_packet * pkt)3917 get_pkt_trunc_len(struct bpf_packet *pkt)
3918 {
3919 struct pktap_header *pktap = (struct pktap_header *) (pkt->bpfp_header);
3920 uint32_t in_pkt_len = 0;
3921 uint32_t out_pkt_len = 0;
3922 uint32_t tlen = 0;
3923 uint32_t pre_adjust; // L2 header not in mbuf or kern_packet
3924
3925 // bpfp_total_length must contain the BPF packet header
3926 assert3u(pkt->bpfp_total_length, >=, pkt->bpfp_header_length);
3927
3928 // The BPF packet header must contain the pktap header
3929 assert3u(pkt->bpfp_header_length, >=, pktap->pth_length);
3930
3931 // The pre frame length (L2 header) must be contained in the packet
3932 assert3u(pkt->bpfp_total_length, >=, pktap->pth_length + pktap->pth_frame_pre_length);
3933
3934 /*
3935 * pktap->pth_frame_pre_length is the L2 header length and accounts
3936 * for both L2 header in the packet payload and pre_adjust.
3937 *
3938 * pre_adjust represents an adjustment for a pseudo L2 header that is not
3939 * part of packet payload -- not in the mbuf or kern_packet -- and comes
3940 * just after the pktap header.
3941 *
3942 * pktap->pth_length is the size of the pktap header (exclude pre_adjust)
3943 *
3944 * pkt->bpfp_header_length is (pktap->pth_length + pre_adjust)
3945 */
3946 pre_adjust = (uint32_t)(pkt->bpfp_header_length - pktap->pth_length);
3947
3948 if (pktap->pth_iftype == IFT_ETHER) {
3949 /*
3950 * We need to parse the Ethernet header to find the network layer
3951 * protocol
3952 */
3953 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pre_adjust);
3954
3955 out_pkt_len = get_ether_trunc_len(pkt, 0, in_pkt_len);
3956
3957 tlen = pktap->pth_length + pre_adjust + out_pkt_len;
3958 } else {
3959 /*
3960 * For other interface types, we only know to parse IPv4 and IPv6.
3961 *
3962 * To get to the beginning of the IPv4 or IPv6 packet, we need to to skip
3963 * over the L2 header that is the actual packet payload (mbuf or kern_packet)
3964 */
3965 uint32_t off; // offset past the L2 header in the actual packet payload
3966
3967 off = pktap->pth_frame_pre_length - pre_adjust;
3968
3969 in_pkt_len = (uint32_t)(pkt->bpfp_total_length - pktap->pth_length - pktap->pth_frame_pre_length);
3970
3971 if (pktap->pth_protocol_family == AF_INET) {
3972 out_pkt_len = get_ip_trunc_len(pkt, off, in_pkt_len);
3973 } else if (pktap->pth_protocol_family == AF_INET6) {
3974 out_pkt_len = get_ip6_trunc_len(pkt, off, in_pkt_len);
3975 } else {
3976 out_pkt_len = MIN(BPF_MIN_PKT_SIZE, in_pkt_len);
3977 }
3978 tlen = pktap->pth_length + pktap->pth_frame_pre_length + out_pkt_len;
3979 }
3980
3981 // Verify we do not overflow the buffer
3982 if (__improbable(tlen > pkt->bpfp_total_length)) {
3983 bool do_panic = bpf_debug != 0 ? true : false;
3984
3985 #if DEBUG
3986 do_panic = true;
3987 #endif /* DEBUG */
3988 if (do_panic) {
3989 panic("%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
3990 __func__, __LINE__,
3991 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
3992 } else {
3993 os_log(OS_LOG_DEFAULT,
3994 "%s:%d tlen %u > bpfp_total_length %lu bpfp_header_length %lu pth_frame_pre_length %u pre_adjust %u in_pkt_len %u out_pkt_len %u",
3995 __func__, __LINE__,
3996 tlen, pkt->bpfp_total_length, pkt->bpfp_header_length, pktap->pth_frame_pre_length, pre_adjust, in_pkt_len, out_pkt_len);
3997 }
3998 bpf_trunc_overflow += 1;
3999 tlen = (uint32_t)pkt->bpfp_total_length;
4000 }
4001
4002 return tlen;
4003 }
4004
4005 static uint8_t
get_common_prefix_size(const void * __bidi_indexable a,const void * __bidi_indexable b,uint8_t max_bytes)4006 get_common_prefix_size(const void *__bidi_indexable a, const void *__bidi_indexable b, uint8_t max_bytes)
4007 {
4008 uint8_t max_words = max_bytes >> 2;
4009 const uint32_t *x = (const uint32_t *)a;
4010 const uint32_t *y = (const uint32_t *)b;
4011 uint8_t i;
4012
4013 for (i = 0; i < max_words; i++) {
4014 if (x[i] != y[i]) {
4015 break;
4016 }
4017 }
4018 return (uint8_t)(i << 2);
4019 }
4020
4021 /*
4022 * Move the packet data from interface memory (pkt) into the
4023 * store buffer. Return 1 if it's time to wakeup a listener (buffer full),
4024 * otherwise 0.
4025 */
4026 static void
catchpacket(struct bpf_d * d,struct bpf_packet * pkt,u_int snaplen,int outbound)4027 catchpacket(struct bpf_d *d, struct bpf_packet * pkt,
4028 u_int snaplen, int outbound)
4029 {
4030 struct bpf_hdr *hp;
4031 struct bpf_hdr_ext *ehp;
4032 uint32_t totlen, curlen;
4033 uint32_t hdrlen, caplen;
4034 int do_wakeup = 0;
4035 u_char *payload;
4036 struct timeval tv = { .tv_sec = 0, .tv_usec = 0 };
4037
4038 hdrlen = (d->bd_flags & BPF_EXTENDED_HDR) ? d->bd_bif->bif_exthdrlen :
4039 (d->bd_flags & BPF_COMP_REQ) ? d->bd_bif->bif_comphdrlen:
4040 d->bd_bif->bif_hdrlen;
4041 /*
4042 * Figure out how many bytes to move. If the packet is
4043 * greater or equal to the snapshot length, transfer that
4044 * much. Otherwise, transfer the whole packet (unless
4045 * we hit the buffer size limit).
4046 */
4047 totlen = hdrlen + MIN(snaplen, (int)pkt->bpfp_total_length);
4048 if (totlen > d->bd_bufsize) {
4049 totlen = d->bd_bufsize;
4050 }
4051
4052 if (hdrlen > totlen) {
4053 return;
4054 }
4055
4056 /*
4057 * Round up the end of the previous packet to the next longword.
4058 */
4059 curlen = BPF_WORDALIGN(d->bd_slen);
4060 if (curlen + totlen > d->bd_bufsize) {
4061 /*
4062 * This packet will overflow the storage buffer.
4063 * Rotate the buffers if we can, then wakeup any
4064 * pending reads.
4065 *
4066 * We cannot rotate buffers if a read is in progress
4067 * so drop the packet
4068 */
4069 if (d->bd_hbuf_read) {
4070 ++d->bd_dcount;
4071 return;
4072 }
4073
4074 if (d->bd_fbuf == NULL) {
4075 if (d->bd_headdrop == 0) {
4076 /*
4077 * We haven't completed the previous read yet,
4078 * so drop the packet.
4079 */
4080 ++d->bd_dcount;
4081 return;
4082 }
4083 /*
4084 * Drop the hold buffer as it contains older packets
4085 */
4086 d->bd_dcount += d->bd_hcnt;
4087 d->bd_fbuf = d->bd_hbuf;
4088 ROTATE_BUFFERS(d);
4089 } else {
4090 ROTATE_BUFFERS(d);
4091 }
4092 do_wakeup = 1;
4093 curlen = 0;
4094 } else if (d->bd_immediate || d->bd_state == BPF_TIMED_OUT) {
4095 /*
4096 * Immediate mode is set, or the read timeout has
4097 * already expired during a select call. A packet
4098 * arrived, so the reader should be woken up.
4099 */
4100 do_wakeup = 1;
4101 }
4102
4103 /*
4104 * Append the bpf header.
4105 */
4106 if (d->bd_tstamp != BPF_T_NONE) {
4107 microtime(&tv);
4108 }
4109 if (d->bd_flags & BPF_EXTENDED_HDR) {
4110 ehp = (struct bpf_hdr_ext *)(void *)(d->bd_sbuf + curlen);
4111 memset(ehp, 0, sizeof(*ehp));
4112 ehp->bh_tstamp.tv_sec = (int)tv.tv_sec;
4113 ehp->bh_tstamp.tv_usec = tv.tv_usec;
4114
4115 ehp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
4116 ehp->bh_hdrlen = (u_short)hdrlen;
4117 caplen = ehp->bh_caplen = totlen - hdrlen;
4118 payload = (u_char *)ehp + hdrlen;
4119
4120 if (outbound) {
4121 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_OUT;
4122 } else {
4123 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_DIR_IN;
4124 }
4125
4126 if (pkt->bpfp_type == BPF_PACKET_TYPE_MBUF) {
4127 mbuf_ref_t m = pkt->bpfp_mbuf;
4128
4129 if (m->m_pkthdr.pkt_ext_flags & PKTF_EXT_ULPN) {
4130 ehp->bh_pktflags |= BPF_PKTFLAGS_ULPN;
4131 }
4132
4133 if (outbound) {
4134 /* only do lookups on non-raw INPCB */
4135 if ((m->m_pkthdr.pkt_flags & (PKTF_FLOW_ID |
4136 PKTF_FLOW_LOCALSRC | PKTF_FLOW_RAWSOCK)) ==
4137 (PKTF_FLOW_ID | PKTF_FLOW_LOCALSRC) &&
4138 m->m_pkthdr.pkt_flowsrc == FLOWSRC_INPCB) {
4139 ehp->bh_flowid = m->m_pkthdr.pkt_flowid;
4140 if (m->m_pkthdr.pkt_proto == IPPROTO_TCP) {
4141 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_TCP;
4142 } else if (m->m_pkthdr.pkt_proto == IPPROTO_UDP) {
4143 ehp->bh_flags |= BPF_HDR_EXT_FLAGS_UDP;
4144 }
4145 }
4146 ehp->bh_svc = so_svc2tc(m->m_pkthdr.pkt_svc);
4147 if (m->m_pkthdr.pkt_flags & PKTF_TCP_REXMT) {
4148 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
4149 }
4150 if (m->m_pkthdr.pkt_flags & PKTF_START_SEQ) {
4151 ehp->bh_pktflags |= BPF_PKTFLAGS_START_SEQ;
4152 }
4153 if (m->m_pkthdr.pkt_flags & PKTF_LAST_PKT) {
4154 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
4155 }
4156 if (m->m_pkthdr.pkt_flags & PKTF_VALID_UNSENT_DATA) {
4157 ehp->bh_unsent_bytes =
4158 m->m_pkthdr.bufstatus_if;
4159 ehp->bh_unsent_snd =
4160 m->m_pkthdr.bufstatus_sndbuf;
4161 }
4162 ehp->bh_comp_gencnt = m->m_pkthdr.comp_gencnt;
4163 } else {
4164 if (m->m_pkthdr.pkt_flags & PKTF_WAKE_PKT) {
4165 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
4166 }
4167 }
4168 #if SKYWALK
4169 } else {
4170 kern_packet_t kern_pkt = pkt->bpfp_pkt;
4171 packet_flowid_t flowid = 0;
4172
4173 if (kern_packet_get_ulpn_flag(kern_pkt)) {
4174 ehp->bh_pktflags |= BPF_PKTFLAGS_ULPN;
4175 }
4176
4177 if (outbound) {
4178 /*
4179 * Note: pp_init() asserts that kern_packet_svc_class_t is equivalent
4180 * to mbuf_svc_class_t
4181 */
4182 ehp->bh_svc = so_svc2tc((mbuf_svc_class_t)kern_packet_get_service_class(kern_pkt));
4183 if (kern_packet_get_transport_retransmit(kern_pkt)) {
4184 ehp->bh_pktflags |= BPF_PKTFLAGS_TCP_REXMT;
4185 }
4186 if (kern_packet_get_transport_last_packet(kern_pkt)) {
4187 ehp->bh_pktflags |= BPF_PKTFLAGS_LAST_PKT;
4188 }
4189 kern_packet_get_compression_generation_count(kern_pkt, &ehp->bh_comp_gencnt);
4190 } else {
4191 if (kern_packet_get_wake_flag(kern_pkt)) {
4192 ehp->bh_pktflags |= BPF_PKTFLAGS_WAKE_PKT;
4193 }
4194 }
4195 ehp->bh_trace_tag = kern_packet_get_trace_tag(kern_pkt);
4196 if (kern_packet_get_flowid(kern_pkt, &flowid) == 0) {
4197 ehp->bh_flowid = flowid;
4198 }
4199 #endif /* SKYWALK */
4200 }
4201 } else {
4202 hp = (struct bpf_hdr *)(void *)(d->bd_sbuf + curlen);
4203 memset(hp, 0, BPF_WORDALIGN(sizeof(*hp)));
4204 hp->bh_tstamp.tv_sec = (int)tv.tv_sec;
4205 hp->bh_tstamp.tv_usec = tv.tv_usec;
4206 hp->bh_datalen = (bpf_u_int32)pkt->bpfp_total_length;
4207 hp->bh_hdrlen = (u_short)hdrlen;
4208 caplen = hp->bh_caplen = totlen - hdrlen;
4209 payload = (u_char *)hp + hdrlen;
4210 }
4211 if (d->bd_flags & BPF_COMP_REQ) {
4212 uint8_t common_prefix_size = 0;
4213 uint8_t copy_len = MIN((uint8_t)caplen, BPF_HDR_COMP_LEN_MAX);
4214
4215 copy_bpf_packet(pkt, (uint8_t *__bidi_indexable)d->bd_prev_fbuf, copy_len);
4216
4217 if (d->bd_prev_slen != 0) {
4218 common_prefix_size = get_common_prefix_size(d->bd_prev_fbuf,
4219 d->bd_prev_sbuf, MIN(copy_len, d->bd_prev_slen));
4220 }
4221
4222 if (d->bd_flags & BPF_COMP_ENABLED) {
4223 assert3u(caplen, >=, common_prefix_size);
4224 copy_bpf_packet_offset(pkt, payload, caplen - common_prefix_size,
4225 common_prefix_size);
4226 d->bd_slen = curlen + totlen - common_prefix_size;
4227 } else {
4228 copy_bpf_packet(pkt, payload, caplen);
4229 d->bd_slen = curlen + totlen;
4230 }
4231
4232 /*
4233 * Update the caplen only if compression is enabled -- the caller
4234 * must pay attention to bpf_hdr_comp_enable
4235 */
4236 if (d->bd_flags & BPF_EXTENDED_HDR) {
4237 ehp->bh_complen = common_prefix_size;
4238 if (d->bd_flags & BPF_COMP_ENABLED) {
4239 ehp->bh_caplen -= common_prefix_size;
4240 }
4241 } else {
4242 struct bpf_comp_hdr *hcp;
4243
4244 hcp = (struct bpf_comp_hdr *)(void *)(d->bd_sbuf + curlen);
4245 hcp->bh_complen = common_prefix_size;
4246 if (d->bd_flags & BPF_COMP_ENABLED) {
4247 hcp->bh_caplen -= common_prefix_size;
4248 }
4249 }
4250
4251 if (common_prefix_size > 0) {
4252 d->bd_bcs.bcs_total_compressed_prefix_size += common_prefix_size;
4253 if (common_prefix_size > d->bd_bcs.bcs_max_compressed_prefix_size) {
4254 d->bd_bcs.bcs_max_compressed_prefix_size = common_prefix_size;
4255 }
4256 d->bd_bcs.bcs_count_compressed_prefix += 1;
4257 } else {
4258 d->bd_bcs.bcs_count_no_common_prefix += 1;
4259 }
4260
4261 /* The current compression buffer becomes the previous one */
4262 caddr_t tmp = d->bd_prev_sbuf;
4263 d->bd_prev_sbuf = d->bd_prev_fbuf;
4264 d->bd_prev_slen = copy_len;
4265 d->bd_prev_fbuf = tmp;
4266 } else {
4267 /*
4268 * Copy the packet data into the store buffer and update its length.
4269 */
4270 copy_bpf_packet(pkt, payload, caplen);
4271 d->bd_slen = curlen + totlen;
4272 }
4273 d->bd_scnt += 1;
4274 d->bd_bcs.bcs_total_hdr_size += pkt->bpfp_header_length;
4275 d->bd_bcs.bcs_total_size += caplen;
4276
4277 if (do_wakeup) {
4278 bpf_wakeup(d);
4279 }
4280 }
4281
4282 static void
bpf_freebufs(struct bpf_d * d)4283 bpf_freebufs(struct bpf_d *d)
4284 {
4285 if (d->bd_sbuf != NULL) {
4286 kfree_data_addr(d->bd_sbuf);
4287 }
4288 if (d->bd_hbuf != NULL) {
4289 kfree_data_addr(d->bd_hbuf);
4290 }
4291 if (d->bd_fbuf != NULL) {
4292 kfree_data_addr(d->bd_fbuf);
4293 }
4294
4295 if (d->bd_prev_sbuf != NULL) {
4296 kfree_data_addr(d->bd_prev_sbuf);
4297 }
4298 if (d->bd_prev_fbuf != NULL) {
4299 kfree_data_addr(d->bd_prev_fbuf);
4300 }
4301 }
4302 /*
4303 * Initialize all nonzero fields of a descriptor.
4304 */
4305 static int
bpf_allocbufs(struct bpf_d * d)4306 bpf_allocbufs(struct bpf_d *d)
4307 {
4308 bpf_freebufs(d);
4309
4310 d->bd_fbuf = kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
4311 if (d->bd_fbuf == NULL) {
4312 goto nobufs;
4313 }
4314
4315 d->bd_sbuf = kalloc_data(d->bd_bufsize, Z_WAITOK | Z_ZERO);
4316 if (d->bd_sbuf == NULL) {
4317 goto nobufs;
4318 }
4319 d->bd_slen = 0;
4320 d->bd_hlen = 0;
4321 d->bd_scnt = 0;
4322 d->bd_hcnt = 0;
4323
4324 d->bd_prev_slen = 0;
4325 if (d->bd_flags & BPF_COMP_REQ) {
4326 d->bd_prev_sbuf = kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
4327 if (d->bd_prev_sbuf == NULL) {
4328 goto nobufs;
4329 }
4330 d->bd_prev_fbuf = kalloc_data(BPF_HDR_COMP_LEN_MAX, Z_WAITOK | Z_ZERO);
4331 if (d->bd_prev_fbuf == NULL) {
4332 goto nobufs;
4333 }
4334 }
4335 return 0;
4336 nobufs:
4337 bpf_freebufs(d);
4338 return ENOMEM;
4339 }
4340
4341 /*
4342 * Free buffers currently in use by a descriptor.
4343 * Called on close.
4344 */
4345 static void
bpf_freed(struct bpf_d * d)4346 bpf_freed(struct bpf_d *d)
4347 {
4348 /*
4349 * We don't need to lock out interrupts since this descriptor has
4350 * been detached from its interface and it yet hasn't been marked
4351 * free.
4352 */
4353 if (d->bd_hbuf_read || d->bd_hbuf_write) {
4354 panic("bpf buffer freed during read/write");
4355 }
4356
4357 bpf_freebufs(d);
4358
4359 if (d->bd_filter) {
4360 kfree_data_addr_sized_by(d->bd_filter, d->bd_filter_len);
4361 }
4362 }
4363
4364 /*
4365 * Attach an interface to bpf. driverp is a pointer to a (struct bpf_if *)
4366 * in the driver's softc; dlt is the link layer type; hdrlen is the fixed
4367 * size of the link header (variable length headers not yet supported).
4368 */
4369 void
bpfattach(struct ifnet * ifp,u_int dlt,u_int hdrlen)4370 bpfattach(struct ifnet *ifp, u_int dlt, u_int hdrlen)
4371 {
4372 bpf_attach(ifp, dlt, hdrlen, NULL, NULL);
4373 }
4374
4375 errno_t
bpf_attach(ifnet_t ifp,u_int32_t dlt,u_int32_t hdrlen,bpf_send_func send,bpf_tap_func tap)4376 bpf_attach(
4377 ifnet_t ifp,
4378 u_int32_t dlt,
4379 u_int32_t hdrlen,
4380 bpf_send_func send,
4381 bpf_tap_func tap)
4382 {
4383 struct bpf_if *bp;
4384 struct bpf_if *bp_new;
4385 struct bpf_if *bp_before_first = NULL;
4386 struct bpf_if *bp_first = NULL;
4387 struct bpf_if *bp_last = NULL;
4388 boolean_t found;
4389
4390 /*
4391 * Z_NOFAIL will cause a panic if the allocation fails
4392 */
4393 bp_new = kalloc_type(struct bpf_if, Z_WAITOK | Z_NOFAIL | Z_ZERO);
4394
4395 lck_mtx_lock(bpf_mlock);
4396
4397 /*
4398 * Check if this interface/dlt is already attached. Remember the
4399 * first and last attachment for this interface, as well as the
4400 * element before the first attachment.
4401 */
4402 found = FALSE;
4403 for (bp = bpf_iflist; bp != NULL; bp = bp->bif_next) {
4404 if (bp->bif_ifp != ifp) {
4405 if (bp_first != NULL) {
4406 /* no more elements for this interface */
4407 break;
4408 }
4409 bp_before_first = bp;
4410 } else {
4411 if (bp->bif_dlt == dlt) {
4412 found = TRUE;
4413 break;
4414 }
4415 if (bp_first == NULL) {
4416 bp_first = bp;
4417 }
4418 bp_last = bp;
4419 }
4420 }
4421 if (found) {
4422 lck_mtx_unlock(bpf_mlock);
4423 os_log_error(OS_LOG_DEFAULT,
4424 "bpfattach - %s with dlt %d is already attached",
4425 if_name(ifp), dlt);
4426 kfree_type(struct bpf_if, bp_new);
4427 return EEXIST;
4428 }
4429
4430 bp_new->bif_ifp = ifp;
4431 bp_new->bif_dlt = dlt;
4432 bp_new->bif_send = send;
4433 bp_new->bif_tap = tap;
4434
4435 if (bp_first == NULL) {
4436 /* No other entries for this ifp */
4437 bp_new->bif_next = bpf_iflist;
4438 bpf_iflist = bp_new;
4439 } else {
4440 if (ifnet_type(ifp) == IFT_ETHER && dlt == DLT_EN10MB) {
4441 /* Make this the first entry for this interface */
4442 if (bp_before_first != NULL) {
4443 /* point the previous to us */
4444 bp_before_first->bif_next = bp_new;
4445 } else {
4446 /* we're the new head */
4447 bpf_iflist = bp_new;
4448 }
4449 bp_new->bif_next = bp_first;
4450 } else {
4451 /* Add this after the last entry for this interface */
4452 bp_new->bif_next = bp_last->bif_next;
4453 bp_last->bif_next = bp_new;
4454 }
4455 }
4456
4457 /*
4458 * Compute the length of the bpf header. This is not necessarily
4459 * equal to SIZEOF_BPF_HDR because we want to insert spacing such
4460 * that the network layer header begins on a longword boundary (for
4461 * performance reasons and to alleviate alignment restrictions).
4462 */
4463 bp_new->bif_hdrlen = BPF_WORDALIGN(hdrlen + SIZEOF_BPF_HDR) - hdrlen;
4464 bp_new->bif_exthdrlen = BPF_WORDALIGN(hdrlen +
4465 sizeof(struct bpf_hdr_ext)) - hdrlen;
4466 bp_new->bif_comphdrlen = BPF_WORDALIGN(hdrlen +
4467 sizeof(struct bpf_comp_hdr)) - hdrlen;
4468
4469 /* Take a reference on the interface */
4470 ifnet_reference(ifp);
4471
4472 lck_mtx_unlock(bpf_mlock);
4473
4474 return 0;
4475 }
4476
4477 /*
4478 * Detach bpf from an interface. This involves detaching each descriptor
4479 * associated with the interface, and leaving bd_bif NULL. Notify each
4480 * descriptor as it's detached so that any sleepers wake up and get
4481 * ENXIO.
4482 */
4483 void
bpfdetach(struct ifnet * ifp)4484 bpfdetach(struct ifnet *ifp)
4485 {
4486 struct bpf_if *bp, *bp_prev, *bp_next;
4487 struct bpf_d *d;
4488
4489 if (bpf_debug != 0) {
4490 os_log(OS_LOG_DEFAULT, "%s: %s", __func__, if_name(ifp));
4491 }
4492
4493 lck_mtx_lock(bpf_mlock);
4494
4495 /*
4496 * Build the list of devices attached to that interface
4497 * that we need to free while keeping the lock to maintain
4498 * the integrity of the interface list
4499 */
4500 bp_prev = NULL;
4501 for (bp = bpf_iflist; bp != NULL; bp = bp_next) {
4502 bp_next = bp->bif_next;
4503
4504 if (ifp != bp->bif_ifp) {
4505 bp_prev = bp;
4506 continue;
4507 }
4508 /* Unlink from the interface list */
4509 if (bp_prev) {
4510 bp_prev->bif_next = bp->bif_next;
4511 } else {
4512 bpf_iflist = bp->bif_next;
4513 }
4514
4515 /* Detach the devices attached to the interface */
4516 while ((d = bp->bif_dlist) != NULL) {
4517 /*
4518 * Take an extra reference to prevent the device
4519 * from being freed when bpf_detachd() releases
4520 * the reference for the interface list
4521 */
4522 bpf_acquire_d(d);
4523
4524 /*
4525 * Wait for active read and writes to complete
4526 */
4527 while (d->bd_hbuf_read || d->bd_hbuf_write) {
4528 msleep((caddr_t)d, bpf_mlock, PRINET, "bpfdetach", NULL);
4529 }
4530
4531 bpf_detachd(d, current_proc());
4532 bpf_wakeup(d);
4533 bpf_release_d(d);
4534 }
4535 ifnet_release(ifp);
4536 }
4537
4538 lck_mtx_unlock(bpf_mlock);
4539 }
4540
4541 void
bpf_init(__unused void * unused)4542 bpf_init(__unused void *unused)
4543 {
4544 int maj;
4545
4546 /* bpf_comp_hdr is an overlay of bpf_hdr */
4547 static_assert(BPF_WORDALIGN(sizeof(struct bpf_hdr)) == BPF_WORDALIGN(sizeof(struct bpf_comp_hdr)));
4548
4549 /* compression length must fits in a byte */
4550 static_assert(BPF_HDR_COMP_LEN_MAX <= UCHAR_MAX);
4551
4552 (void) PE_parse_boot_argn("bpf_hdr_comp", &bpf_hdr_comp_enable,
4553 sizeof(bpf_hdr_comp_enable));
4554
4555 if (bpf_devsw_installed == 0) {
4556 bpf_devsw_installed = 1;
4557 maj = cdevsw_add(CDEV_MAJOR, &bpf_cdevsw);
4558 if (maj == -1) {
4559 bpf_devsw_installed = 0;
4560 os_log_error(OS_LOG_DEFAULT,
4561 "bpf_init: failed to allocate a major number");
4562 return;
4563 }
4564
4565 for (int i = 0; i < NBPFILTER; i++) {
4566 bpf_make_dev_t(maj);
4567 }
4568 }
4569 }
4570
4571 static int
4572 sysctl_bpf_maxbufsize SYSCTL_HANDLER_ARGS
4573 {
4574 #pragma unused(arg1, arg2)
4575 int i, err;
4576
4577 i = bpf_maxbufsize;
4578
4579 err = sysctl_handle_int(oidp, &i, 0, req);
4580 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4581 return err;
4582 }
4583
4584 if (i < 0 || i > BPF_BUFSIZE_CAP) {
4585 i = BPF_BUFSIZE_CAP;
4586 }
4587
4588 bpf_maxbufsize = i;
4589 return err;
4590 }
4591
4592 static int
4593 sysctl_bpf_bufsize_cap SYSCTL_HANDLER_ARGS
4594 {
4595 #pragma unused(arg1, arg2)
4596 int i, err;
4597
4598 i = BPF_BUFSIZE_CAP;
4599
4600 err = sysctl_handle_int(oidp, &i, 0, req);
4601 if (err != 0 || req->newptr == USER_ADDR_NULL) {
4602 return err;
4603 }
4604
4605 return err;
4606 }
4607
4608 /*
4609 * Fill filter statistics
4610 */
4611 static void
bpfstats_fill_xbpf(struct xbpf_d * d,struct bpf_d * bd)4612 bpfstats_fill_xbpf(struct xbpf_d *d, struct bpf_d *bd)
4613 {
4614 LCK_MTX_ASSERT(bpf_mlock, LCK_MTX_ASSERT_OWNED);
4615
4616 d->bd_structsize = sizeof(struct xbpf_d);
4617 d->bd_promisc = bd->bd_promisc != 0 ? 1 : 0;
4618 d->bd_immediate = d->bd_immediate != 0 ? 1 : 0;
4619 d->bd_hdrcmplt = bd->bd_hdrcmplt != 0 ? 1 : 0;
4620 d->bd_async = bd->bd_async != 0 ? 1 : 0;
4621 d->bd_headdrop = bd->bd_headdrop != 0 ? 1 : 0;
4622 d->bd_direction = (uint8_t)bd->bd_direction;
4623 d->bh_compreq = bd->bd_flags & BPF_COMP_REQ ? 1 : 0;
4624 d->bh_compenabled = bd->bd_flags & BPF_COMP_ENABLED ? 1 : 0;
4625 d->bd_exthdr = bd->bd_flags & BPF_EXTENDED_HDR ? 1 : 0;
4626 d->bd_trunc = bd->bd_flags & BPF_TRUNCATE ? 1 : 0;
4627 d->bd_pkthdrv2 = bd->bd_flags & BPF_PKTHDRV2 ? 1 : 0;
4628 d->bd_batch_write = bd->bd_flags & BPF_BATCH_WRITE ? 1 : 0;
4629 d->bd_divert_in = bd->bd_flags & BPF_DIVERT_IN ? 1 : 0;
4630
4631 d->bd_dev_minor = (uint8_t)bd->bd_dev_minor;
4632
4633 d->bd_sig = bd->bd_sig;
4634
4635 d->bd_rcount = bd->bd_rcount;
4636 d->bd_dcount = bd->bd_dcount;
4637 d->bd_fcount = bd->bd_fcount;
4638 d->bd_wcount = bd->bd_wcount;
4639 d->bd_wdcount = bd->bd_wdcount;
4640 d->bd_slen = bd->bd_slen;
4641 d->bd_hlen = bd->bd_hlen;
4642 d->bd_bufsize = bd->bd_bufsize;
4643 d->bd_pid = bd->bd_pid;
4644 if (bd->bd_bif != NULL && bd->bd_bif->bif_ifp != NULL) {
4645 strlcpy(d->bd_ifname,
4646 bd->bd_bif->bif_ifp->if_xname, IFNAMSIZ);
4647 }
4648
4649 d->bd_comp_count = bd->bd_bcs.bcs_count_compressed_prefix;
4650 d->bd_comp_size = bd->bd_bcs.bcs_total_compressed_prefix_size;
4651
4652 d->bd_scnt = bd->bd_scnt;
4653 d->bd_hcnt = bd->bd_hcnt;
4654
4655 d->bd_read_count = bd->bd_bcs.bcs_total_read;
4656 d->bd_fsize = bd->bd_bcs.bcs_total_size;
4657 }
4658
4659 /*
4660 * Handle `netstat -B' stats request
4661 */
4662 static int
4663 sysctl_bpf_stats SYSCTL_HANDLER_ARGS
4664 {
4665 int error;
4666 struct xbpf_d *xbdbuf;
4667 unsigned int x_cnt;
4668 vm_size_t buf_size;
4669
4670 if (req->oldptr == USER_ADDR_NULL) {
4671 return SYSCTL_OUT(req, 0, nbpfilter * sizeof(struct xbpf_d));
4672 }
4673 if (nbpfilter == 0) {
4674 return SYSCTL_OUT(req, 0, 0);
4675 }
4676 buf_size = req->oldlen;
4677 if (buf_size > BPF_MAX_DEVICES * sizeof(struct xbpf_d)) {
4678 buf_size = BPF_MAX_DEVICES * sizeof(struct xbpf_d);
4679 }
4680 xbdbuf = kalloc_data(buf_size, Z_WAITOK | Z_ZERO);
4681
4682 lck_mtx_lock(bpf_mlock);
4683 if (buf_size < (nbpfilter * sizeof(struct xbpf_d))) {
4684 lck_mtx_unlock(bpf_mlock);
4685 kfree_data(xbdbuf, buf_size);
4686 return ENOMEM;
4687 }
4688 x_cnt = 0;
4689 unsigned int i;
4690
4691 for (i = 0; i < nbpfilter; i++) {
4692 struct bpf_d *bd = bpf_dtab[i];
4693 struct xbpf_d *xbd;
4694
4695 if (bd == NULL || bd == BPF_DEV_RESERVED ||
4696 (bd->bd_flags & BPF_CLOSING) != 0) {
4697 continue;
4698 }
4699 VERIFY(x_cnt < nbpfilter);
4700
4701 xbd = &xbdbuf[x_cnt++];
4702 bpfstats_fill_xbpf(xbd, bd);
4703 }
4704 lck_mtx_unlock(bpf_mlock);
4705
4706 error = SYSCTL_OUT(req, xbdbuf, x_cnt * sizeof(struct xbpf_d));
4707 kfree_data(xbdbuf, buf_size);
4708 return error;
4709 }
4710