1 /*
2 * Copyright (c) 1999-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <stddef.h>
30 #include <stdint.h>
31 #include <sys/queue.h>
32 #include <sys/mcache.h>
33 #include <libkern/OSAtomic.h>
34
35 #include <kern/zalloc.h>
36
37 #include <net/dlil_var_private.h>
38 #include <net/if_var_private.h>
39
40 /*
41 * DLIL device management
42 */
43 int
dlil_if_acquire(uint32_t family,const void * uniqueid __sized_by (uniqueid_len),size_t uniqueid_len,const char * ifxname0 __null_terminated,struct ifnet ** ifp)44 dlil_if_acquire(uint32_t family, const void *uniqueid __sized_by(uniqueid_len),
45 size_t uniqueid_len, const char *ifxname0 __null_terminated, struct ifnet **ifp)
46 {
47 struct ifnet *ifp1 = NULL;
48 struct dlil_ifnet *dlifp1 = NULL;
49 struct dlil_ifnet *dlifp1_saved = NULL;
50 int ret = 0;
51 size_t ifxname_len = strlen(ifxname0);
52 const char *ifxname = __unsafe_forge_bidi_indexable(const char *, ifxname0, ifxname_len);
53 size_t ifp_name_len;
54
55 VERIFY(*ifp == NULL);
56 dlil_if_lock();
57 /*
58 * We absolutely can't have an interface with the same name
59 * in in-use state.
60 * To make sure of that list has to be traversed completely
61 */
62 TAILQ_FOREACH(dlifp1, &dlil_ifnet_head, dl_if_link) {
63 ifp1 = (struct ifnet *)dlifp1;
64 ifp_name_len = strlen(ifp1->if_name);
65 if (IFXNAMSIZ < ifp_name_len) {
66 ifp_name_len = IFXNAMSIZ;
67 }
68
69 if (ifp1->if_family != family) {
70 continue;
71 }
72
73 /*
74 * If interface is in use, return EBUSY if either unique id
75 * or interface extended names are the same
76 */
77 lck_mtx_lock(&dlifp1->dl_if_lock);
78 /*
79 * Note: compare the lengths to avoid least prefix match.
80 */
81 if (ifxname_len == ifp_name_len &&
82 strlcmp(ifxname, ifp1->if_xname, ifxname_len) == 0 &&
83 (dlifp1->dl_if_flags & DLIF_INUSE) != 0) {
84 lck_mtx_unlock(&dlifp1->dl_if_lock);
85 ret = EBUSY;
86 goto end;
87 }
88
89 if (uniqueid_len != 0 &&
90 uniqueid_len == dlifp1->dl_if_uniqueid_len &&
91 bcmp(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len) == 0) {
92 if ((dlifp1->dl_if_flags & DLIF_INUSE) != 0) {
93 lck_mtx_unlock(&dlifp1->dl_if_lock);
94 ret = EBUSY;
95 goto end;
96 }
97 if (dlifp1_saved == NULL) {
98 /* cache the first match */
99 dlifp1_saved = dlifp1;
100 }
101 /*
102 * Do not break or jump to end as we have to traverse
103 * the whole list to ensure there are no name collisions
104 */
105 }
106 lck_mtx_unlock(&dlifp1->dl_if_lock);
107 }
108
109 /* If there's an interface that can be recycled, use that */
110 if (dlifp1_saved != NULL) {
111 lck_mtx_lock(&dlifp1_saved->dl_if_lock);
112 if ((dlifp1_saved->dl_if_flags & DLIF_INUSE) != 0) {
113 /* some other thread got in ahead of us */
114 lck_mtx_unlock(&dlifp1_saved->dl_if_lock);
115 ret = EBUSY;
116 goto end;
117 }
118 dlifp1_saved->dl_if_flags |= (DLIF_INUSE | DLIF_REUSE);
119 lck_mtx_unlock(&dlifp1_saved->dl_if_lock);
120 *ifp = (struct ifnet *)dlifp1_saved;
121 dlil_if_ref(*ifp);
122 goto end;
123 }
124
125 /* no interface found, allocate a new one */
126 dlifp1 = dlif_ifnet_alloc();
127
128 if (uniqueid_len) {
129 void *new_uniqueid = kalloc_data(uniqueid_len,
130 Z_WAITOK);
131 if (new_uniqueid == NULL) {
132 dlif_ifnet_free(dlifp1);
133 ret = ENOMEM;
134 goto end;
135 }
136 dlifp1->dl_if_uniqueid_len = uniqueid_len;
137 dlifp1->dl_if_uniqueid = new_uniqueid;
138
139 bcopy(uniqueid, dlifp1->dl_if_uniqueid, uniqueid_len);
140 }
141
142 ifp1 = (struct ifnet *)dlifp1;
143 dlifp1->dl_if_flags = DLIF_INUSE;
144 ifp1->if_name = tsnprintf(dlifp1->dl_if_namestorage, sizeof(dlifp1->dl_if_namestorage), "");
145 ifp1->if_xname = tsnprintf(dlifp1->dl_if_xnamestorage, sizeof(dlifp1->dl_if_xnamestorage), "");
146
147 /* initialize interface description */
148 ifp1->if_desc.ifd_maxlen = IF_DESCSIZE;
149 ifp1->if_desc.ifd_len = 0;
150 ifp1->if_desc.ifd_desc = dlifp1->dl_if_descstorage;
151
152 #if SKYWALK
153 LIST_INIT(&ifp1->if_netns_tokens);
154 #endif /* SKYWALK */
155
156 if ((ret = dlil_alloc_local_stats(ifp1)) != 0) {
157 DLIL_PRINTF("%s: failed to allocate if local stats, "
158 "error: %d\n", __func__, ret);
159 /* This probably shouldn't be fatal */
160 ret = 0;
161 }
162
163 lck_mtx_init(&dlifp1->dl_if_lock, &ifnet_lock_group, &ifnet_lock_attr);
164 lck_rw_init(&ifp1->if_lock, &ifnet_lock_group, &ifnet_lock_attr);
165 lck_mtx_init(&ifp1->if_ref_lock, &ifnet_lock_group, &ifnet_lock_attr);
166 lck_mtx_init(&ifp1->if_flt_lock, &ifnet_lock_group, &ifnet_lock_attr);
167 lck_mtx_init(&ifp1->if_addrconfig_lock, &ifnet_lock_group,
168 &ifnet_lock_attr);
169 lck_rw_init(&ifp1->if_llreach_lock, &ifnet_lock_group, &ifnet_lock_attr);
170 #if INET
171 lck_rw_init(&ifp1->if_inetdata_lock, &ifnet_lock_group,
172 &ifnet_lock_attr);
173 ifp1->if_inetdata = NULL;
174 #endif
175 lck_mtx_init(&ifp1->if_inet6_ioctl_lock, &ifnet_lock_group, &ifnet_lock_attr);
176 ifp1->if_inet6_ioctl_busy = FALSE;
177 lck_rw_init(&ifp1->if_inet6data_lock, &ifnet_lock_group,
178 &ifnet_lock_attr);
179 ifp1->if_inet6data = NULL;
180 lck_rw_init(&ifp1->if_link_status_lock, &ifnet_lock_group,
181 &ifnet_lock_attr);
182 ifp1->if_link_status = NULL;
183 lck_mtx_init(&ifp1->if_delegate_lock, &ifnet_lock_group, &ifnet_lock_attr);
184
185 /* for send data paths */
186 lck_mtx_init(&ifp1->if_start_lock, &ifnet_snd_lock_group,
187 &ifnet_lock_attr);
188 lck_mtx_init(&ifp1->if_cached_route_lock, &ifnet_snd_lock_group,
189 &ifnet_lock_attr);
190
191 /* for receive data paths */
192 lck_mtx_init(&ifp1->if_poll_lock, &ifnet_rcv_lock_group,
193 &ifnet_lock_attr);
194
195 /* thread call allocation is done with sleeping zalloc */
196 ifp1->if_dt_tcall = thread_call_allocate_with_options(dlil_dt_tcall_fn,
197 ifp1, THREAD_CALL_PRIORITY_KERNEL, THREAD_CALL_OPTIONS_ONCE);
198 if (ifp1->if_dt_tcall == NULL) {
199 panic_plain("%s: couldn't create if_dt_tcall", __func__);
200 /* NOTREACHED */
201 }
202
203 TAILQ_INSERT_TAIL(&dlil_ifnet_head, dlifp1, dl_if_link);
204
205 *ifp = ifp1;
206 dlil_if_ref(*ifp);
207
208 end:
209 dlil_if_unlock();
210
211 VERIFY(dlifp1 == NULL || (IS_P2ALIGNED(dlifp1, sizeof(u_int64_t)) &&
212 IS_P2ALIGNED(&ifp1->if_data, sizeof(u_int64_t))));
213
214 return ret;
215 }
216
217 /*
218 * Stats management.
219 */
220 void
dlil_input_stats_add(const struct ifnet_stat_increment_param * s,struct dlil_threading_info * inp,struct ifnet * ifp,boolean_t poll)221 dlil_input_stats_add(const struct ifnet_stat_increment_param *s,
222 struct dlil_threading_info *inp, struct ifnet *ifp, boolean_t poll)
223 {
224 struct ifnet_stat_increment_param *d = &inp->dlth_stats;
225
226 if (s->packets_in != 0) {
227 d->packets_in += s->packets_in;
228 }
229 if (s->bytes_in != 0) {
230 d->bytes_in += s->bytes_in;
231 }
232 if (s->errors_in != 0) {
233 d->errors_in += s->errors_in;
234 }
235
236 if (s->packets_out != 0) {
237 d->packets_out += s->packets_out;
238 }
239 if (s->bytes_out != 0) {
240 d->bytes_out += s->bytes_out;
241 }
242 if (s->errors_out != 0) {
243 d->errors_out += s->errors_out;
244 }
245
246 if (s->collisions != 0) {
247 d->collisions += s->collisions;
248 }
249 if (s->dropped != 0) {
250 d->dropped += s->dropped;
251 }
252
253 if (poll) {
254 PKTCNTR_ADD(&ifp->if_poll_tstats, s->packets_in, s->bytes_in);
255 }
256 }
257
258 boolean_t
dlil_input_stats_sync(struct ifnet * ifp,struct dlil_threading_info * inp)259 dlil_input_stats_sync(struct ifnet *ifp, struct dlil_threading_info *inp)
260 {
261 struct ifnet_stat_increment_param *s = &inp->dlth_stats;
262
263 /*
264 * Use of atomic operations is unavoidable here because
265 * these stats may also be incremented elsewhere via KPIs.
266 */
267 if (s->packets_in != 0) {
268 os_atomic_add(&ifp->if_data.ifi_ipackets, s->packets_in, relaxed);
269 s->packets_in = 0;
270 }
271 if (s->bytes_in != 0) {
272 os_atomic_add(&ifp->if_data.ifi_ibytes, s->bytes_in, relaxed);
273 s->bytes_in = 0;
274 }
275 if (s->errors_in != 0) {
276 os_atomic_add(&ifp->if_data.ifi_ierrors, s->errors_in, relaxed);
277 s->errors_in = 0;
278 }
279
280 if (s->packets_out != 0) {
281 os_atomic_add(&ifp->if_data.ifi_opackets, s->packets_out, relaxed);
282 s->packets_out = 0;
283 }
284 if (s->bytes_out != 0) {
285 os_atomic_add(&ifp->if_data.ifi_obytes, s->bytes_out, relaxed);
286 s->bytes_out = 0;
287 }
288 if (s->errors_out != 0) {
289 os_atomic_add(&ifp->if_data.ifi_oerrors, s->errors_out, relaxed);
290 s->errors_out = 0;
291 }
292
293 if (s->collisions != 0) {
294 os_atomic_add(&ifp->if_data.ifi_collisions, s->collisions, relaxed);
295 s->collisions = 0;
296 }
297 if (s->dropped != 0) {
298 os_atomic_add(&ifp->if_data.ifi_iqdrops, s->dropped, relaxed);
299 s->dropped = 0;
300 }
301
302 /*
303 * No need for atomic operations as they are modified here
304 * only from within the DLIL input thread context.
305 */
306 if (ifp->if_poll_tstats.packets != 0) {
307 ifp->if_poll_pstats.ifi_poll_packets += ifp->if_poll_tstats.packets;
308 ifp->if_poll_tstats.packets = 0;
309 }
310 if (ifp->if_poll_tstats.bytes != 0) {
311 ifp->if_poll_pstats.ifi_poll_bytes += ifp->if_poll_tstats.bytes;
312 ifp->if_poll_tstats.bytes = 0;
313 }
314
315 return ifp->if_data_threshold != 0;
316 }
317
318
319 #if SKYWALK
320 errno_t
dlil_set_input_handler(struct ifnet * ifp,dlil_input_func fn)321 dlil_set_input_handler(struct ifnet *ifp, dlil_input_func fn)
322 {
323 return os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_input_dlil),
324 ptrauth_nop_cast(void *, &dlil_input_handler),
325 ptrauth_nop_cast(void *, fn), acq_rel) ? 0 : EBUSY;
326 }
327
328 void
dlil_reset_input_handler(struct ifnet * ifp)329 dlil_reset_input_handler(struct ifnet *ifp)
330 {
331 while (!os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_input_dlil),
332 ptrauth_nop_cast(void *, ifp->if_input_dlil),
333 ptrauth_nop_cast(void *, &dlil_input_handler), acq_rel)) {
334 ;
335 }
336 }
337
338 errno_t
dlil_set_output_handler(struct ifnet * ifp,dlil_output_func fn)339 dlil_set_output_handler(struct ifnet *ifp, dlil_output_func fn)
340 {
341 return os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_output_dlil),
342 ptrauth_nop_cast(void *, &dlil_output_handler),
343 ptrauth_nop_cast(void *, fn), acq_rel) ? 0 : EBUSY;
344 }
345
346 void
dlil_reset_output_handler(struct ifnet * ifp)347 dlil_reset_output_handler(struct ifnet *ifp)
348 {
349 while (!os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_output_dlil),
350 ptrauth_nop_cast(void *, ifp->if_output_dlil),
351 ptrauth_nop_cast(void *, &dlil_output_handler), acq_rel)) {
352 ;
353 }
354 }
355 #endif /* SKYWALK */
356
357 errno_t
dlil_output_handler(struct ifnet * ifp,struct mbuf * m)358 dlil_output_handler(struct ifnet *ifp, struct mbuf *m)
359 {
360 return ifp->if_output(ifp, m);
361 }
362
363 #define MAX_KNOWN_MBUF_CLASS 8
364
365
366 #if SKYWALK
367 errno_t
ifnet_set_output_handler(struct ifnet * ifp,ifnet_output_func fn)368 ifnet_set_output_handler(struct ifnet *ifp, ifnet_output_func fn)
369 {
370 return os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_output),
371 ptrauth_nop_cast(void *, ifp->if_save_output),
372 ptrauth_nop_cast(void *, fn), acq_rel) ? 0 : EBUSY;
373 }
374
375 void
ifnet_reset_output_handler(struct ifnet * ifp)376 ifnet_reset_output_handler(struct ifnet *ifp)
377 {
378 while (!os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_output),
379 ptrauth_nop_cast(void *, ifp->if_output),
380 ptrauth_nop_cast(void *, ifp->if_save_output), acq_rel)) {
381 ;
382 }
383 }
384
385 errno_t
ifnet_set_start_handler(struct ifnet * ifp,ifnet_start_func fn)386 ifnet_set_start_handler(struct ifnet *ifp, ifnet_start_func fn)
387 {
388 return os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_start),
389 ptrauth_nop_cast(void *, ifp->if_save_start),
390 ptrauth_nop_cast(void *, fn), acq_rel) ? 0 : EBUSY;
391 }
392
393 void
ifnet_reset_start_handler(struct ifnet * ifp)394 ifnet_reset_start_handler(struct ifnet *ifp)
395 {
396 while (!os_atomic_cmpxchg(__unsafe_forge_single(void * volatile *, &ifp->if_start),
397 ptrauth_nop_cast(void *, ifp->if_start),
398 ptrauth_nop_cast(void *, ifp->if_save_start), acq_rel)) {
399 ;
400 }
401 }
402 #endif /* SKYWALK */
403