1 /*
2 * Copyright (c) 2012-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * A note on the MPTCP/NECP-interactions:
31 *
32 * MPTCP uses NECP-callbacks to get notified of interface/policy events.
33 * MPTCP registers to these events at the MPTCP-layer for interface-events
34 * through a call to necp_client_register_multipath_cb.
35 * To get per-flow events (aka per TCP-subflow), we register to it with
36 * necp_client_register_socket_flow. Both registrations happen by using the
37 * necp-client-uuid that comes from the app.
38 *
39 * The locking is rather tricky. In general, we expect the lock-ordering to
40 * happen from necp-fd -> necp->client -> mpp_lock.
41 *
42 * There are however some subtleties.
43 *
44 * 1. When registering the multipath_cb, we are holding the mpp_lock. This is
45 * safe, because it is the very first time this MPTCP-connection goes into NECP.
46 * As we go into NECP we take the NECP-locks and thus are guaranteed that no
47 * NECP-locks will deadlock us. Because these NECP-events will also first take
48 * the NECP-locks. Either they win the race and thus won't find our
49 * MPTCP-connection. Or, MPTCP wins the race and thus it will safely install
50 * the callbacks while holding the NECP lock.
51 *
52 * 2. When registering the subflow-callbacks we must unlock the mpp_lock. This,
53 * because we have already registered callbacks and we might race against an
54 * NECP-event that will match on our socket. So, we have to unlock to be safe.
55 *
56 * 3. When removing the multipath_cb, we do it in mp_pcbdispose(). The
57 * so_usecount has reached 0. We must be careful to not remove the mpp_socket
58 * pointers before we unregistered the callback. Because, again we might be
59 * racing against an NECP-event. Unregistering must happen with an unlocked
60 * mpp_lock, because of the lock-ordering constraint. It could be that
61 * before we had a chance to unregister an NECP-event triggers. That's why
62 * we need to check for the so_usecount in mptcp_session_necp_cb. If we get
63 * there while the socket is being garbage-collected, the use-count will go
64 * down to 0 and we exit. Removal of the multipath_cb again happens by taking
65 * the NECP-locks so any running NECP-events will finish first and exit cleanly.
66 *
67 * 4. When removing the subflow-callback, we do it in in_pcbdispose(). Again,
68 * the socket-lock must be unlocked for lock-ordering constraints. This gets a
69 * bit tricky here, as in tcp_garbage_collect we hold the mp_so and so lock.
70 * So, we drop the mp_so-lock as soon as the subflow is unlinked with
71 * mptcp_subflow_del. Then, in in_pcbdispose we drop the subflow-lock.
72 * If an NECP-event was waiting on the lock in mptcp_subflow_necp_cb, when it
73 * gets it, it will realize that the subflow became non-MPTCP and retry (see
74 * tcp_lock). Then it waits again on the subflow-lock. When we drop this lock
75 * in in_pcbdispose, and enter necp_inpcb_dispose, this one will have to wait
76 * for the NECP-lock (held by the other thread that is taking care of the NECP-
77 * event). So, the event now finally gets the subflow-lock and then hits an
78 * so_usecount that is 0 and exits. Eventually, we can remove the subflow from
79 * the NECP callback.
80 */
81
82 #include <sys/param.h>
83 #include <sys/systm.h>
84 #include <sys/kernel.h>
85 #include <sys/mbuf.h>
86 #include <sys/mcache.h>
87 #include <sys/socket.h>
88 #include <sys/socketvar.h>
89 #include <sys/syslog.h>
90 #include <sys/protosw.h>
91
92 #include <kern/zalloc.h>
93 #include <kern/locks.h>
94
95 #include <mach/sdt.h>
96
97 #include <net/if.h>
98 #include <netinet/in.h>
99 #include <netinet/in_var.h>
100 #include <netinet/tcp.h>
101 #include <netinet/tcp_fsm.h>
102 #include <netinet/tcp_seq.h>
103 #include <netinet/tcp_var.h>
104 #include <netinet/mptcp_var.h>
105 #include <netinet/mptcp.h>
106 #include <netinet/mptcp_seq.h>
107 #include <netinet/mptcp_opt.h>
108 #include <netinet/mptcp_timer.h>
109
110 int mptcp_enable = 1;
111 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, enable, CTLFLAG_RW | CTLFLAG_LOCKED,
112 &mptcp_enable, 0, "Enable Multipath TCP Support");
113
114 /*
115 * Number of times to try negotiating MPTCP on SYN retransmissions.
116 * We haven't seen any reports of a middlebox that is dropping all SYN-segments
117 * that have an MPTCP-option. Thus, let's be generous and retransmit it 4 times.
118 */
119 int mptcp_mpcap_retries = 4;
120 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, mptcp_cap_retr,
121 CTLFLAG_RW | CTLFLAG_LOCKED,
122 &mptcp_mpcap_retries, 0, "Number of MP Capable SYN Retries");
123
124 /*
125 * By default, DSS checksum is turned off, revisit if we ever do
126 * MPTCP for non SSL Traffic.
127 */
128 int mptcp_dss_csum = 0;
129 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, dss_csum, CTLFLAG_RW | CTLFLAG_LOCKED,
130 &mptcp_dss_csum, 0, "Enable DSS checksum");
131
132 /*
133 * When mptcp_fail_thresh number of retransmissions are sent, subflow failover
134 * is attempted on a different path.
135 */
136 int mptcp_fail_thresh = 1;
137 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, fail, CTLFLAG_RW | CTLFLAG_LOCKED,
138 &mptcp_fail_thresh, 0, "Failover threshold");
139
140 /*
141 * MPTCP subflows have TCP keepalives set to ON. Set a conservative keeptime
142 * as carrier networks mostly have a 30 minute to 60 minute NAT Timeout.
143 * Some carrier networks have a timeout of 10 or 15 minutes.
144 */
145 int mptcp_subflow_keeptime = 60 * 14;
146 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, keepalive, CTLFLAG_RW | CTLFLAG_LOCKED,
147 &mptcp_subflow_keeptime, 0, "Keepalive in seconds");
148
149 int mptcp_rtthist_rtthresh = 600;
150 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rtthist_thresh, CTLFLAG_RW | CTLFLAG_LOCKED,
151 &mptcp_rtthist_rtthresh, 0, "Rtt threshold");
152
153 int mptcp_rtothresh = 1500;
154 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, rto_thresh, CTLFLAG_RW | CTLFLAG_LOCKED,
155 &mptcp_rtothresh, 0, "RTO threshold");
156
157 /*
158 * Probe the preferred path, when it is not in use
159 */
160 uint32_t mptcp_probeto = 1000;
161 SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probeto, CTLFLAG_RW | CTLFLAG_LOCKED,
162 &mptcp_probeto, 0, "Disable probing by setting to 0");
163
164 uint32_t mptcp_probecnt = 5;
165 SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, probecnt, CTLFLAG_RW | CTLFLAG_LOCKED,
166 &mptcp_probecnt, 0, "Number of probe writes");
167
168 uint32_t mptcp_enable_v1 = 1;
169 SYSCTL_UINT(_net_inet_mptcp, OID_AUTO, enable_v1, CTLFLAG_RW | CTLFLAG_LOCKED,
170 &mptcp_enable_v1, 0, "Enable or disable v1");
171
172 static int
173 sysctl_mptcp_version_check SYSCTL_HANDLER_ARGS
174 {
175 #pragma unused(arg1, arg2)
176 int error;
177 int new_value = *(int *)oidp->oid_arg1;
178 int old_value = *(int *)oidp->oid_arg1;
179
180 error = sysctl_handle_int(oidp, &new_value, 0, req);
181 if (!error) {
182 if (new_value != MPTCP_VERSION_0 && new_value != MPTCP_VERSION_1) {
183 return EINVAL;
184 }
185 *(int *)oidp->oid_arg1 = new_value;
186 }
187
188 os_log(OS_LOG_DEFAULT,
189 "%s:%u sysctl net.inet.tcp.mptcp_preferred_version: %d -> %d)",
190 proc_best_name(current_proc()), proc_selfpid(),
191 old_value, *(int *)oidp->oid_arg1);
192
193 return error;
194 }
195
196 int mptcp_preferred_version = MPTCP_VERSION_1;
197 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, mptcp_preferred_version,
198 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
199 &mptcp_preferred_version, 0, &sysctl_mptcp_version_check, "I", "");
200
201 int mptcp_reass_total_qlen = 0;
202 SYSCTL_INT(_net_inet_mptcp, OID_AUTO, reass_qlen,
203 CTLFLAG_RD | CTLFLAG_LOCKED, &mptcp_reass_total_qlen, 0,
204 "Total number of MPTCP segments in reassembly queues");
205
206 static int
mptcp_reass_present(struct socket * mp_so)207 mptcp_reass_present(struct socket *mp_so)
208 {
209 struct mptses *mpte = mpsotompte(mp_so);
210 struct mptcb *mp_tp = mpte->mpte_mptcb;
211 struct tseg_qent *q;
212 int dowakeup = 0;
213 int flags = 0;
214 int count = 0;
215
216 /*
217 * Present data to user, advancing rcv_nxt through
218 * completed sequence space.
219 */
220 if (mp_tp->mpt_state < MPTCPS_ESTABLISHED) {
221 return flags;
222 }
223 q = LIST_FIRST(&mp_tp->mpt_segq);
224 if (!q || q->tqe_m->m_pkthdr.mp_dsn != mp_tp->mpt_rcvnxt) {
225 return flags;
226 }
227
228 /*
229 * If there is already another thread doing reassembly for this
230 * connection, it is better to let it finish the job --
231 * (radar 16316196)
232 */
233 if (mp_tp->mpt_flags & MPTCPF_REASS_INPROG) {
234 return flags;
235 }
236
237 mp_tp->mpt_flags |= MPTCPF_REASS_INPROG;
238
239 do {
240 mp_tp->mpt_rcvnxt += q->tqe_len;
241 LIST_REMOVE(q, tqe_q);
242 if (mp_so->so_state & SS_CANTRCVMORE) {
243 m_freem(q->tqe_m);
244 } else {
245 flags = !!(q->tqe_m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);
246 if (sbappendstream_rcvdemux(mp_so, q->tqe_m)) {
247 dowakeup = 1;
248 }
249 }
250 zfree(tcp_reass_zone, q);
251 mp_tp->mpt_reassqlen--;
252 count++;
253 q = LIST_FIRST(&mp_tp->mpt_segq);
254 } while (q && q->tqe_m->m_pkthdr.mp_dsn == mp_tp->mpt_rcvnxt);
255 mp_tp->mpt_flags &= ~MPTCPF_REASS_INPROG;
256
257 if (count > 0) {
258 OSAddAtomic(-count, &mptcp_reass_total_qlen);
259 }
260 if (dowakeup) {
261 sorwakeup(mp_so); /* done with socket lock held */
262 }
263 return flags;
264 }
265
266 static int
mptcp_reass(struct socket * mp_so,struct pkthdr * phdr,int * tlenp,struct mbuf * m)267 mptcp_reass(struct socket *mp_so, struct pkthdr *phdr, int *tlenp, struct mbuf *m)
268 {
269 struct mptcb *mp_tp = mpsotomppcb(mp_so)->mpp_pcbe->mpte_mptcb;
270 u_int64_t mb_dsn = phdr->mp_dsn;
271 struct tseg_qent *q;
272 struct tseg_qent *p = NULL;
273 struct tseg_qent *nq;
274 struct tseg_qent *te = NULL;
275 uint32_t qlimit;
276
277 /*
278 * Limit the number of segments in the reassembly queue to prevent
279 * holding on to too many segments (and thus running out of mbufs).
280 * Make sure to let the missing segment through which caused this
281 * queue. Always keep one global queue entry spare to be able to
282 * process the missing segment.
283 */
284 qlimit = MIN(MAX(100, mp_so->so_rcv.sb_hiwat >> 10),
285 (tcp_autorcvbuf_max >> 10));
286 if (mb_dsn != mp_tp->mpt_rcvnxt &&
287 (mp_tp->mpt_reassqlen + 1) >= qlimit) {
288 tcpstat.tcps_mptcp_rcvmemdrop++;
289 m_freem(m);
290 *tlenp = 0;
291 return 0;
292 }
293
294 /* Allocate a new queue entry. If we can't, just drop the pkt. XXX */
295 te = zalloc_flags(tcp_reass_zone, Z_WAITOK | Z_NOFAIL);
296
297 mp_tp->mpt_reassqlen++;
298 OSIncrementAtomic(&mptcp_reass_total_qlen);
299
300 /*
301 * Find a segment which begins after this one does.
302 */
303 LIST_FOREACH(q, &mp_tp->mpt_segq, tqe_q) {
304 if (MPTCP_SEQ_GT(q->tqe_m->m_pkthdr.mp_dsn, mb_dsn)) {
305 break;
306 }
307 p = q;
308 }
309
310 /*
311 * If there is a preceding segment, it may provide some of
312 * our data already. If so, drop the data from the incoming
313 * segment. If it provides all of our data, drop us.
314 */
315 if (p != NULL) {
316 int64_t i;
317 /* conversion to int (in i) handles seq wraparound */
318 i = p->tqe_m->m_pkthdr.mp_dsn + p->tqe_len - mb_dsn;
319 if (i > 0) {
320 if (i >= *tlenp) {
321 tcpstat.tcps_mptcp_rcvduppack++;
322 m_freem(m);
323 zfree(tcp_reass_zone, te);
324 te = NULL;
325 mp_tp->mpt_reassqlen--;
326 OSDecrementAtomic(&mptcp_reass_total_qlen);
327 /*
328 * Try to present any queued data
329 * at the left window edge to the user.
330 * This is needed after the 3-WHS
331 * completes.
332 */
333 goto out;
334 }
335 VERIFY(i <= INT_MAX);
336 m_adj(m, (int)i);
337 *tlenp -= i;
338 phdr->mp_dsn += i;
339 }
340 }
341
342 tcpstat.tcps_mp_oodata++;
343
344 /*
345 * While we overlap succeeding segments trim them or,
346 * if they are completely covered, dequeue them.
347 */
348 while (q) {
349 int64_t i = (mb_dsn + *tlenp) - q->tqe_m->m_pkthdr.mp_dsn;
350 if (i <= 0) {
351 break;
352 }
353
354 if (i < q->tqe_len) {
355 q->tqe_m->m_pkthdr.mp_dsn += i;
356 q->tqe_len -= i;
357
358 VERIFY(i <= INT_MAX);
359 m_adj(q->tqe_m, (int)i);
360 break;
361 }
362
363 nq = LIST_NEXT(q, tqe_q);
364 LIST_REMOVE(q, tqe_q);
365 m_freem(q->tqe_m);
366 zfree(tcp_reass_zone, q);
367 mp_tp->mpt_reassqlen--;
368 OSDecrementAtomic(&mptcp_reass_total_qlen);
369 q = nq;
370 }
371
372 /* Insert the new segment queue entry into place. */
373 te->tqe_m = m;
374 te->tqe_th = NULL;
375 te->tqe_len = *tlenp;
376
377 if (p == NULL) {
378 LIST_INSERT_HEAD(&mp_tp->mpt_segq, te, tqe_q);
379 } else {
380 LIST_INSERT_AFTER(p, te, tqe_q);
381 }
382
383 out:
384 return mptcp_reass_present(mp_so);
385 }
386
387 /*
388 * MPTCP input, called when data has been read from a subflow socket.
389 */
390 void
mptcp_input(struct mptses * mpte,struct mbuf * m)391 mptcp_input(struct mptses *mpte, struct mbuf *m)
392 {
393 struct socket *mp_so;
394 struct mptcb *mp_tp = NULL;
395 int count = 0, wakeup = 0;
396 struct mbuf *save = NULL, *prev = NULL;
397 struct mbuf *freelist = NULL, *tail = NULL;
398
399 if (__improbable((m->m_flags & M_PKTHDR) == 0)) {
400 panic("mbuf invalid: %p", m);
401 }
402
403 mp_so = mptetoso(mpte);
404 mp_tp = mpte->mpte_mptcb;
405
406 socket_lock_assert_owned(mp_so);
407
408 DTRACE_MPTCP(input);
409
410 mp_tp->mpt_rcvwnd = mptcp_sbspace(mp_tp);
411
412 /*
413 * Each mbuf contains MPTCP Data Sequence Map
414 * Process the data for reassembly, delivery to MPTCP socket
415 * client, etc.
416 *
417 */
418 count = mp_so->so_rcv.sb_cc;
419
420 /*
421 * In the degraded fallback case, data is accepted without DSS map
422 */
423 if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {
424 struct mbuf *iter;
425 int mb_dfin;
426 fallback:
427 mb_dfin = 0;
428 mptcp_sbrcv_grow(mp_tp);
429
430 iter = m;
431 while (iter) {
432 if ((iter->m_flags & M_PKTHDR) &&
433 (iter->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN)) {
434 mb_dfin = 1;
435 }
436
437 if ((iter->m_flags & M_PKTHDR) && m_pktlen(iter) == 0) {
438 /* Don't add zero-length packets, so jump it! */
439 if (prev == NULL) {
440 m = iter->m_next;
441 m_free(iter);
442 iter = m;
443 } else {
444 prev->m_next = iter->m_next;
445 m_free(iter);
446 iter = prev->m_next;
447 }
448
449 /* It was a zero-length packet so next one must be a pkthdr */
450 VERIFY(iter == NULL || iter->m_flags & M_PKTHDR);
451 } else {
452 prev = iter;
453 iter = iter->m_next;
454 }
455 }
456
457 /*
458 * assume degraded flow as this may be the first packet
459 * without DSS, and the subflow state is not updated yet.
460 */
461 if (sbappendstream_rcvdemux(mp_so, m)) {
462 sorwakeup(mp_so);
463 }
464
465 DTRACE_MPTCP5(receive__degraded, struct mbuf *, m,
466 struct socket *, mp_so,
467 struct sockbuf *, &mp_so->so_rcv,
468 struct sockbuf *, &mp_so->so_snd,
469 struct mptses *, mpte);
470 count = mp_so->so_rcv.sb_cc - count;
471
472 mp_tp->mpt_rcvnxt += count;
473
474 if (mb_dfin) {
475 mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
476 socantrcvmore(mp_so);
477 }
478 return;
479 }
480
481 do {
482 u_int64_t mb_dsn;
483 int32_t mb_datalen;
484 int64_t todrop;
485 int mb_dfin = 0;
486
487 VERIFY(m->m_flags & M_PKTHDR);
488
489 /* If fallback occurs, mbufs will not have PKTF_MPTCP set */
490 if (!(m->m_pkthdr.pkt_flags & PKTF_MPTCP)) {
491 goto fallback;
492 }
493
494 save = m->m_next;
495 /*
496 * A single TCP packet formed of multiple mbufs
497 * holds DSS mapping in the first mbuf of the chain.
498 * Other mbufs in the chain may have M_PKTHDR set
499 * even though they belong to the same TCP packet
500 * and therefore use the DSS mapping stored in the
501 * first mbuf of the mbuf chain. mptcp_input() can
502 * get an mbuf chain with multiple TCP packets.
503 */
504 while (save && (!(save->m_flags & M_PKTHDR) ||
505 !(save->m_pkthdr.pkt_flags & PKTF_MPTCP))) {
506 prev = save;
507 save = save->m_next;
508 }
509 if (prev) {
510 prev->m_next = NULL;
511 } else {
512 m->m_next = NULL;
513 }
514
515 mb_dsn = m->m_pkthdr.mp_dsn;
516 mb_datalen = m->m_pkthdr.mp_rlen;
517
518 todrop = (mb_dsn + mb_datalen) - (mp_tp->mpt_rcvnxt + mp_tp->mpt_rcvwnd);
519 if (todrop > 0) {
520 tcpstat.tcps_mptcp_rcvpackafterwin++;
521
522 os_log_info(mptcp_log_handle, "%s - %lx: dropping dsn %u dlen %u rcvnxt %u rcvwnd %u todrop %lld\n",
523 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mpte),
524 (uint32_t)mb_dsn, mb_datalen, (uint32_t)mp_tp->mpt_rcvnxt,
525 mp_tp->mpt_rcvwnd, todrop);
526
527 if (todrop >= mb_datalen) {
528 if (freelist == NULL) {
529 freelist = m;
530 } else {
531 tail->m_next = m;
532 }
533
534 if (prev != NULL) {
535 tail = prev;
536 } else {
537 tail = m;
538 }
539
540 m = save;
541 prev = save = NULL;
542 continue;
543 } else {
544 VERIFY(todrop <= INT_MAX);
545 m_adj(m, (int)-todrop);
546 mb_datalen -= todrop;
547 m->m_pkthdr.mp_rlen -= todrop;
548 }
549
550 /*
551 * We drop from the right edge of the mbuf, thus the
552 * DATA_FIN is dropped as well
553 */
554 m->m_pkthdr.pkt_flags &= ~PKTF_MPTCP_DFIN;
555 }
556
557 if (MPTCP_SEQ_LT(mb_dsn, mp_tp->mpt_rcvnxt)) {
558 if (MPTCP_SEQ_LEQ((mb_dsn + mb_datalen),
559 mp_tp->mpt_rcvnxt)) {
560 if (freelist == NULL) {
561 freelist = m;
562 } else {
563 tail->m_next = m;
564 }
565
566 if (prev != NULL) {
567 tail = prev;
568 } else {
569 tail = m;
570 }
571
572 m = save;
573 prev = save = NULL;
574 continue;
575 } else {
576 VERIFY((mp_tp->mpt_rcvnxt - mb_dsn) <= INT_MAX);
577 m_adj(m, (int)(mp_tp->mpt_rcvnxt - mb_dsn));
578 mb_datalen -= (mp_tp->mpt_rcvnxt - mb_dsn);
579 mb_dsn = mp_tp->mpt_rcvnxt;
580 VERIFY(mb_datalen >= 0 && mb_datalen <= USHRT_MAX);
581 m->m_pkthdr.mp_rlen = (uint16_t)mb_datalen;
582 m->m_pkthdr.mp_dsn = mb_dsn;
583 }
584 }
585
586 if (MPTCP_SEQ_GT(mb_dsn, mp_tp->mpt_rcvnxt) ||
587 !LIST_EMPTY(&mp_tp->mpt_segq)) {
588 mb_dfin = mptcp_reass(mp_so, &m->m_pkthdr, &mb_datalen, m);
589
590 goto next;
591 }
592 mb_dfin = !!(m->m_pkthdr.pkt_flags & PKTF_MPTCP_DFIN);
593
594 mptcp_sbrcv_grow(mp_tp);
595
596 if (sbappendstream_rcvdemux(mp_so, m)) {
597 wakeup = 1;
598 }
599
600 DTRACE_MPTCP6(receive, struct mbuf *, m, struct socket *, mp_so,
601 struct sockbuf *, &mp_so->so_rcv,
602 struct sockbuf *, &mp_so->so_snd,
603 struct mptses *, mpte,
604 struct mptcb *, mp_tp);
605 count = mp_so->so_rcv.sb_cc - count;
606 tcpstat.tcps_mp_rcvtotal++;
607 tcpstat.tcps_mp_rcvbytes += count;
608
609 mp_tp->mpt_rcvnxt += count;
610
611 next:
612 if (mb_dfin) {
613 mptcp_close_fsm(mp_tp, MPCE_RECV_DATA_FIN);
614 socantrcvmore(mp_so);
615 }
616 m = save;
617 prev = save = NULL;
618 count = mp_so->so_rcv.sb_cc;
619 } while (m);
620
621 if (freelist) {
622 m_freem(freelist);
623 }
624
625 if (wakeup) {
626 sorwakeup(mp_so);
627 }
628 }
629
630 boolean_t
mptcp_can_send_more(struct mptcb * mp_tp,boolean_t ignore_reinject)631 mptcp_can_send_more(struct mptcb *mp_tp, boolean_t ignore_reinject)
632 {
633 struct socket *mp_so = mptetoso(mp_tp->mpt_mpte);
634
635 /*
636 * Always send if there is data in the reinject-queue.
637 */
638 if (!ignore_reinject && mp_tp->mpt_mpte->mpte_reinjectq) {
639 return TRUE;
640 }
641
642 /*
643 * Don't send, if:
644 *
645 * 1. snd_nxt >= snd_max : Means, basically everything has been sent.
646 * Except when using TFO, we might be doing a 0-byte write.
647 * 2. snd_una + snd_wnd <= snd_nxt: No space in the receiver's window
648 * 3. snd_nxt + 1 == snd_max and we are closing: A DATA_FIN is scheduled.
649 */
650
651 if (!(mp_so->so_flags1 & SOF1_PRECONNECT_DATA) && MPTCP_SEQ_GEQ(mp_tp->mpt_sndnxt, mp_tp->mpt_sndmax)) {
652 return FALSE;
653 }
654
655 if (MPTCP_SEQ_LEQ(mp_tp->mpt_snduna + mp_tp->mpt_sndwnd, mp_tp->mpt_sndnxt)) {
656 return FALSE;
657 }
658
659 if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax && mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) {
660 return FALSE;
661 }
662
663 if (mp_tp->mpt_state >= MPTCPS_FIN_WAIT_2) {
664 return FALSE;
665 }
666
667 return TRUE;
668 }
669
670 /*
671 * MPTCP output.
672 */
673 int
mptcp_output(struct mptses * mpte)674 mptcp_output(struct mptses *mpte)
675 {
676 struct mptcb *mp_tp;
677 struct mptsub *mpts;
678 struct mptsub *mpts_tried = NULL;
679 struct socket *mp_so;
680 struct mptsub *preferred_mpts = NULL;
681 uint64_t old_snd_nxt;
682 int error = 0;
683
684 mp_so = mptetoso(mpte);
685 mp_tp = mpte->mpte_mptcb;
686
687 socket_lock_assert_owned(mp_so);
688
689 if (mp_so->so_flags & SOF_DEFUNCT) {
690 return 0;
691 }
692
693 VERIFY(!(mpte->mpte_mppcb->mpp_flags & MPP_WUPCALL));
694 mpte->mpte_mppcb->mpp_flags |= MPP_WUPCALL;
695
696 old_snd_nxt = mp_tp->mpt_sndnxt;
697 while (mptcp_can_send_more(mp_tp, FALSE)) {
698 /* get the "best" subflow to be used for transmission */
699 mpts = mptcp_get_subflow(mpte, &preferred_mpts);
700 if (mpts == NULL) {
701 break;
702 }
703
704 /* In case there's just one flow, we reattempt later */
705 if (mpts_tried != NULL &&
706 (mpts == mpts_tried || (mpts->mpts_flags & MPTSF_FAILINGOVER))) {
707 mpts_tried->mpts_flags &= ~MPTSF_FAILINGOVER;
708 mpts_tried->mpts_flags |= MPTSF_ACTIVE;
709 mptcp_start_timer(mpte, MPTT_REXMT);
710 break;
711 }
712
713 /*
714 * Automatic sizing of send socket buffer. Increase the send
715 * socket buffer size if all of the following criteria are met
716 * 1. the receiver has enough buffer space for this data
717 * 2. send buffer is filled to 7/8th with data (so we actually
718 * have data to make use of it);
719 */
720 if ((mp_so->so_snd.sb_flags & (SB_AUTOSIZE | SB_TRIM)) == SB_AUTOSIZE &&
721 tcp_cansbgrow(&mp_so->so_snd)) {
722 if ((mp_tp->mpt_sndwnd / 4 * 5) >= mp_so->so_snd.sb_hiwat &&
723 mp_so->so_snd.sb_cc >= (mp_so->so_snd.sb_hiwat / 8 * 7)) {
724 if (sbreserve(&mp_so->so_snd,
725 min(mp_so->so_snd.sb_hiwat + tcp_autosndbuf_inc,
726 tcp_autosndbuf_max)) == 1) {
727 mp_so->so_snd.sb_idealsize = mp_so->so_snd.sb_hiwat;
728 }
729 }
730 }
731
732 DTRACE_MPTCP3(output, struct mptses *, mpte, struct mptsub *, mpts,
733 struct socket *, mp_so);
734 error = mptcp_subflow_output(mpte, mpts, 0);
735 if (error) {
736 /* can be a temporary loss of source address or other error */
737 mpts->mpts_flags |= MPTSF_FAILINGOVER;
738 mpts->mpts_flags &= ~MPTSF_ACTIVE;
739 mpts_tried = mpts;
740 if (error != ECANCELED) {
741 os_log_error(mptcp_log_handle, "%s - %lx: Error = %d mpts_flags %#x\n",
742 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mpte),
743 error, mpts->mpts_flags);
744 }
745 break;
746 }
747 /* The model is to have only one active flow at a time */
748 mpts->mpts_flags |= MPTSF_ACTIVE;
749 mpts->mpts_probesoon = mpts->mpts_probecnt = 0;
750
751 /* Allows us to update the smoothed rtt */
752 if (mptcp_probeto && mpts != preferred_mpts && preferred_mpts != NULL) {
753 if (preferred_mpts->mpts_probesoon) {
754 if ((tcp_now - preferred_mpts->mpts_probesoon) > mptcp_probeto) {
755 mptcp_subflow_output(mpte, preferred_mpts, MPTCP_SUBOUT_PROBING);
756 if (preferred_mpts->mpts_probecnt >= mptcp_probecnt) {
757 preferred_mpts->mpts_probesoon = 0;
758 preferred_mpts->mpts_probecnt = 0;
759 }
760 }
761 } else {
762 preferred_mpts->mpts_probesoon = tcp_now;
763 preferred_mpts->mpts_probecnt = 0;
764 }
765 }
766
767 if (mpte->mpte_active_sub == NULL) {
768 mpte->mpte_active_sub = mpts;
769 } else if (mpte->mpte_active_sub != mpts) {
770 mpte->mpte_active_sub->mpts_flags &= ~MPTSF_ACTIVE;
771 mpte->mpte_active_sub = mpts;
772
773 mptcpstats_inc_switch(mpte, mpts);
774 }
775 }
776
777 if (mp_tp->mpt_state > MPTCPS_CLOSE_WAIT) {
778 if (mp_tp->mpt_sndnxt + 1 == mp_tp->mpt_sndmax &&
779 mp_tp->mpt_snduna == mp_tp->mpt_sndnxt) {
780 mptcp_finish_usrclosed(mpte);
781 }
782 }
783
784 mptcp_handle_deferred_upcalls(mpte->mpte_mppcb, MPP_WUPCALL);
785
786 /* subflow errors should not be percolated back up */
787 return 0;
788 }
789
790
791 static struct mptsub *
mptcp_choose_subflow(struct mptsub * mpts,struct mptsub * curbest,int * currtt)792 mptcp_choose_subflow(struct mptsub *mpts, struct mptsub *curbest, int *currtt)
793 {
794 struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
795
796 /*
797 * Lower RTT? Take it, if it's our first one, or
798 * it doesn't has any loss, or the current one has
799 * loss as well.
800 */
801 if (tp->t_srtt && *currtt > tp->t_srtt &&
802 (curbest == NULL || tp->t_rxtshift == 0 ||
803 sototcpcb(curbest->mpts_socket)->t_rxtshift)) {
804 *currtt = tp->t_srtt;
805 return mpts;
806 }
807
808 /*
809 * If we find a subflow without loss, take it always!
810 */
811 if (curbest &&
812 sototcpcb(curbest->mpts_socket)->t_rxtshift &&
813 tp->t_rxtshift == 0) {
814 *currtt = tp->t_srtt;
815 return mpts;
816 }
817
818 return curbest != NULL ? curbest : mpts;
819 }
820
821 static struct mptsub *
mptcp_return_subflow(struct mptsub * mpts)822 mptcp_return_subflow(struct mptsub *mpts)
823 {
824 if (mpts && mptcp_subflow_cwnd_space(mpts->mpts_socket) <= 0) {
825 return NULL;
826 }
827
828 return mpts;
829 }
830
831 static boolean_t
mptcp_subflow_is_slow(struct mptses * mpte,struct mptsub * mpts)832 mptcp_subflow_is_slow(struct mptses *mpte, struct mptsub *mpts)
833 {
834 struct tcpcb *tp = sototcpcb(mpts->mpts_socket);
835 int fail_thresh = mptcp_fail_thresh;
836
837 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER || mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
838 fail_thresh *= 2;
839 }
840
841 return tp->t_rxtshift >= fail_thresh &&
842 (mptetoso(mpte)->so_snd.sb_cc || mpte->mpte_reinjectq);
843 }
844
845 /*
846 * Return the most eligible subflow to be used for sending data.
847 */
848 struct mptsub *
mptcp_get_subflow(struct mptses * mpte,struct mptsub ** preferred)849 mptcp_get_subflow(struct mptses *mpte, struct mptsub **preferred)
850 {
851 struct tcpcb *besttp, *secondtp;
852 struct inpcb *bestinp, *secondinp;
853 struct mptsub *mpts;
854 struct mptsub *best = NULL;
855 struct mptsub *second_best = NULL;
856 int exp_rtt = INT_MAX, cheap_rtt = INT_MAX;
857
858 /*
859 * First Step:
860 * Choose the best subflow for cellular and non-cellular interfaces.
861 */
862
863 TAILQ_FOREACH(mpts, &mpte->mpte_subflows, mpts_entry) {
864 struct socket *so = mpts->mpts_socket;
865 struct tcpcb *tp = sototcpcb(so);
866 struct inpcb *inp = sotoinpcb(so);
867
868 /*
869 * First, the hard conditions to reject subflows
870 * (e.g., not connected,...)
871 */
872 if (inp->inp_last_outifp == NULL) {
873 continue;
874 }
875
876 if (INP_WAIT_FOR_IF_FEEDBACK(inp)) {
877 continue;
878 }
879
880 /* There can only be one subflow in degraded state */
881 if (mpts->mpts_flags & MPTSF_MP_DEGRADED) {
882 best = mpts;
883 break;
884 }
885
886 /*
887 * If this subflow is waiting to finally send, do it!
888 */
889 if (so->so_flags1 & SOF1_PRECONNECT_DATA) {
890 return mptcp_return_subflow(mpts);
891 }
892
893 /*
894 * Only send if the subflow is MP_CAPABLE. The exceptions to
895 * this rule (degraded or TFO) have been taken care of above.
896 */
897 if (!(mpts->mpts_flags & MPTSF_MP_CAPABLE)) {
898 continue;
899 }
900
901 if ((so->so_state & SS_ISDISCONNECTED) ||
902 !(so->so_state & SS_ISCONNECTED) ||
903 !TCPS_HAVEESTABLISHED(tp->t_state) ||
904 tp->t_state > TCPS_CLOSE_WAIT) {
905 continue;
906 }
907
908 /*
909 * Second, the soft conditions to find the subflow with best
910 * conditions for each set (aka cellular vs non-cellular)
911 */
912 if (IFNET_IS_CELLULAR(inp->inp_last_outifp)) {
913 second_best = mptcp_choose_subflow(mpts, second_best,
914 &exp_rtt);
915 } else {
916 best = mptcp_choose_subflow(mpts, best, &cheap_rtt);
917 }
918 }
919
920 /*
921 * If there is no preferred or backup subflow, and there is no active
922 * subflow use the last usable subflow.
923 */
924 if (best == NULL) {
925 return mptcp_return_subflow(second_best);
926 }
927
928 if (second_best == NULL) {
929 return mptcp_return_subflow(best);
930 }
931
932 besttp = sototcpcb(best->mpts_socket);
933 bestinp = sotoinpcb(best->mpts_socket);
934 secondtp = sototcpcb(second_best->mpts_socket);
935 secondinp = sotoinpcb(second_best->mpts_socket);
936
937 if (preferred != NULL) {
938 *preferred = mptcp_return_subflow(best);
939 }
940
941 /*
942 * Second Step: Among best and second_best. Choose the one that is
943 * most appropriate for this particular service-type.
944 */
945 if (mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
946 return mptcp_return_subflow(best);
947 } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER) {
948 /*
949 * Only handover if Symptoms tells us to do so.
950 */
951 if (!IFNET_IS_CELLULAR(bestinp->inp_last_outifp) &&
952 mptcp_wifi_quality_for_session(mpte) != MPTCP_WIFI_QUALITY_GOOD &&
953 mptcp_subflow_is_slow(mpte, best)) {
954 return mptcp_return_subflow(second_best);
955 }
956
957 return mptcp_return_subflow(best);
958 } else if (mpte->mpte_svctype == MPTCP_SVCTYPE_INTERACTIVE) {
959 int rtt_thresh = mptcp_rtthist_rtthresh << TCP_RTT_SHIFT;
960 int rto_thresh = mptcp_rtothresh;
961
962 /* Adjust with symptoms information */
963 if (!IFNET_IS_CELLULAR(bestinp->inp_last_outifp) &&
964 mptcp_wifi_quality_for_session(mpte) != MPTCP_WIFI_QUALITY_GOOD) {
965 rtt_thresh /= 2;
966 rto_thresh /= 2;
967 }
968
969 if (besttp->t_srtt && secondtp->t_srtt &&
970 besttp->t_srtt >= rtt_thresh &&
971 secondtp->t_srtt < rtt_thresh) {
972 tcpstat.tcps_mp_sel_rtt++;
973 return mptcp_return_subflow(second_best);
974 }
975
976 if (mptcp_subflow_is_slow(mpte, best) &&
977 secondtp->t_rxtshift == 0) {
978 return mptcp_return_subflow(second_best);
979 }
980
981 /* Compare RTOs, select second_best if best's rto exceeds rtothresh */
982 if (besttp->t_rxtcur && secondtp->t_rxtcur &&
983 besttp->t_rxtcur >= rto_thresh &&
984 secondtp->t_rxtcur < rto_thresh) {
985 tcpstat.tcps_mp_sel_rto++;
986
987 return mptcp_return_subflow(second_best);
988 }
989
990 /*
991 * None of the above conditions for sending on the secondary
992 * were true. So, let's schedule on the best one, if he still
993 * has some space in the congestion-window.
994 */
995 return mptcp_return_subflow(best);
996 } else if (mpte->mpte_svctype >= MPTCP_SVCTYPE_AGGREGATE) {
997 struct mptsub *tmp;
998
999 /*
1000 * We only care about RTT when aggregating
1001 */
1002 if (besttp->t_srtt > secondtp->t_srtt) {
1003 tmp = best;
1004 best = second_best;
1005 besttp = secondtp;
1006 bestinp = secondinp;
1007
1008 second_best = tmp;
1009 secondtp = sototcpcb(second_best->mpts_socket);
1010 secondinp = sotoinpcb(second_best->mpts_socket);
1011 }
1012
1013 /* Is there still space in the congestion window? */
1014 if (mptcp_subflow_cwnd_space(bestinp->inp_socket) <= 0) {
1015 return mptcp_return_subflow(second_best);
1016 }
1017
1018 return mptcp_return_subflow(best);
1019 } else {
1020 panic("Unknown service-type configured for MPTCP");
1021 }
1022
1023 return NULL;
1024 }
1025
1026 void
mptcp_close_fsm(struct mptcb * mp_tp,uint32_t event)1027 mptcp_close_fsm(struct mptcb *mp_tp, uint32_t event)
1028 {
1029 struct socket *mp_so = mptetoso(mp_tp->mpt_mpte);
1030
1031 socket_lock_assert_owned(mp_so);
1032
1033 DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp,
1034 uint32_t, event);
1035
1036 switch (mp_tp->mpt_state) {
1037 case MPTCPS_CLOSED:
1038 case MPTCPS_LISTEN:
1039 mp_tp->mpt_state = MPTCPS_TERMINATE;
1040 break;
1041
1042 case MPTCPS_ESTABLISHED:
1043 if (event == MPCE_CLOSE) {
1044 mp_tp->mpt_state = MPTCPS_FIN_WAIT_1;
1045 mp_tp->mpt_sndmax += 1; /* adjust for Data FIN */
1046 } else if (event == MPCE_RECV_DATA_FIN) {
1047 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1048 mp_tp->mpt_state = MPTCPS_CLOSE_WAIT;
1049 }
1050 break;
1051
1052 case MPTCPS_CLOSE_WAIT:
1053 if (event == MPCE_CLOSE) {
1054 mp_tp->mpt_state = MPTCPS_LAST_ACK;
1055 mp_tp->mpt_sndmax += 1; /* adjust for Data FIN */
1056 }
1057 break;
1058
1059 case MPTCPS_FIN_WAIT_1:
1060 if (event == MPCE_RECV_DATA_ACK) {
1061 mp_tp->mpt_state = MPTCPS_FIN_WAIT_2;
1062 } else if (event == MPCE_RECV_DATA_FIN) {
1063 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1064 mp_tp->mpt_state = MPTCPS_CLOSING;
1065 }
1066 break;
1067
1068 case MPTCPS_CLOSING:
1069 if (event == MPCE_RECV_DATA_ACK) {
1070 mp_tp->mpt_state = MPTCPS_TIME_WAIT;
1071 }
1072 break;
1073
1074 case MPTCPS_LAST_ACK:
1075 if (event == MPCE_RECV_DATA_ACK) {
1076 mptcp_close(mp_tp->mpt_mpte, mp_tp);
1077 }
1078 break;
1079
1080 case MPTCPS_FIN_WAIT_2:
1081 if (event == MPCE_RECV_DATA_FIN) {
1082 mp_tp->mpt_rcvnxt += 1; /* adj remote data FIN */
1083 mp_tp->mpt_state = MPTCPS_TIME_WAIT;
1084 }
1085 break;
1086
1087 case MPTCPS_TIME_WAIT:
1088 case MPTCPS_TERMINATE:
1089 break;
1090
1091 default:
1092 VERIFY(0);
1093 /* NOTREACHED */
1094 }
1095 DTRACE_MPTCP2(state__change, struct mptcb *, mp_tp,
1096 uint32_t, event);
1097 }
1098
1099 /* If you change this function, match up mptcp_update_rcv_state_f */
1100 void
mptcp_update_dss_rcv_state(struct mptcp_dsn_opt * dss_info,struct tcpcb * tp,uint16_t csum)1101 mptcp_update_dss_rcv_state(struct mptcp_dsn_opt *dss_info, struct tcpcb *tp,
1102 uint16_t csum)
1103 {
1104 struct mptcb *mp_tp = tptomptp(tp);
1105 u_int64_t full_dsn = 0;
1106
1107 NTOHL(dss_info->mdss_dsn);
1108 NTOHL(dss_info->mdss_subflow_seqn);
1109 NTOHS(dss_info->mdss_data_len);
1110
1111 /* XXX for autosndbuf grow sb here */
1112 MPTCP_EXTEND_DSN(mp_tp->mpt_rcvnxt, dss_info->mdss_dsn, full_dsn);
1113 mptcp_update_rcv_state_meat(mp_tp, tp,
1114 full_dsn, dss_info->mdss_subflow_seqn, dss_info->mdss_data_len,
1115 csum);
1116 }
1117
1118 void
mptcp_update_rcv_state_meat(struct mptcb * mp_tp,struct tcpcb * tp,u_int64_t full_dsn,u_int32_t seqn,u_int16_t mdss_data_len,uint16_t csum)1119 mptcp_update_rcv_state_meat(struct mptcb *mp_tp, struct tcpcb *tp,
1120 u_int64_t full_dsn, u_int32_t seqn, u_int16_t mdss_data_len,
1121 uint16_t csum)
1122 {
1123 if (mdss_data_len == 0) {
1124 os_log_error(mptcp_log_handle, "%s - %lx: Infinite Mapping.\n",
1125 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mp_tp->mpt_mpte));
1126
1127 if ((mp_tp->mpt_flags & MPTCPF_CHECKSUM) && (csum != 0)) {
1128 os_log_error(mptcp_log_handle, "%s - %lx: Bad checksum %x \n",
1129 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mp_tp->mpt_mpte), csum);
1130 }
1131 mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
1132 return;
1133 }
1134
1135 mptcp_notify_mpready(tp->t_inpcb->inp_socket);
1136
1137 tp->t_rcv_map.mpt_dsn = full_dsn;
1138 tp->t_rcv_map.mpt_sseq = seqn;
1139 tp->t_rcv_map.mpt_len = mdss_data_len;
1140 tp->t_rcv_map.mpt_csum = csum;
1141 tp->t_mpflags |= TMPF_EMBED_DSN;
1142 }
1143
1144
1145 static uint16_t
mptcp_input_csum(struct tcpcb * tp,struct mbuf * m,uint64_t dsn,uint32_t sseq,uint16_t dlen,uint16_t csum,int dfin)1146 mptcp_input_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn, uint32_t sseq,
1147 uint16_t dlen, uint16_t csum, int dfin)
1148 {
1149 struct mptcb *mp_tp = tptomptp(tp);
1150 int real_len = dlen - dfin;
1151 uint32_t sum = 0;
1152
1153 VERIFY(real_len >= 0);
1154
1155 if (mp_tp == NULL) {
1156 return 0;
1157 }
1158
1159 if (!(mp_tp->mpt_flags & MPTCPF_CHECKSUM)) {
1160 return 0;
1161 }
1162
1163 if (tp->t_mpflags & TMPF_TCP_FALLBACK) {
1164 return 0;
1165 }
1166
1167 /*
1168 * The remote side may send a packet with fewer bytes than the
1169 * claimed DSS checksum length.
1170 */
1171 if ((int)m_length2(m, NULL) < real_len) {
1172 return 0xffff;
1173 }
1174
1175 if (real_len != 0) {
1176 sum = m_sum16(m, 0, real_len);
1177 }
1178
1179 sum += in_pseudo64(htonll(dsn), htonl(sseq), htons(dlen) + csum);
1180 ADDCARRY(sum);
1181
1182 DTRACE_MPTCP3(checksum__result, struct tcpcb *, tp, struct mbuf *, m,
1183 uint32_t, sum);
1184
1185 return ~sum & 0xffff;
1186 }
1187
1188 /*
1189 * MPTCP Checksum support
1190 * The checksum is calculated whenever the MPTCP DSS option is included
1191 * in the TCP packet. The checksum includes the sum of the MPTCP psuedo
1192 * header and the actual data indicated by the length specified in the
1193 * DSS option.
1194 */
1195
1196 int
mptcp_validate_csum(struct tcpcb * tp,struct mbuf * m,uint64_t dsn,uint32_t sseq,uint16_t dlen,uint16_t csum,int dfin)1197 mptcp_validate_csum(struct tcpcb *tp, struct mbuf *m, uint64_t dsn,
1198 uint32_t sseq, uint16_t dlen, uint16_t csum, int dfin)
1199 {
1200 uint16_t mptcp_csum;
1201
1202 mptcp_csum = mptcp_input_csum(tp, m, dsn, sseq, dlen, csum, dfin);
1203 if (mptcp_csum) {
1204 tp->t_mpflags |= TMPF_SND_MPFAIL;
1205 mptcp_notify_mpfail(tp->t_inpcb->inp_socket);
1206 m_freem(m);
1207 tcpstat.tcps_mp_badcsum++;
1208 return -1;
1209 }
1210 return 0;
1211 }
1212
1213 uint16_t
mptcp_output_csum(struct mbuf * m,uint64_t dss_val,uint32_t sseq,uint16_t dlen)1214 mptcp_output_csum(struct mbuf *m, uint64_t dss_val, uint32_t sseq, uint16_t dlen)
1215 {
1216 uint32_t sum = 0;
1217
1218 if (dlen) {
1219 sum = m_sum16(m, 0, dlen);
1220 }
1221
1222 dss_val = mptcp_hton64(dss_val);
1223 sseq = htonl(sseq);
1224 dlen = htons(dlen);
1225 sum += in_pseudo64(dss_val, sseq, dlen);
1226
1227 ADDCARRY(sum);
1228 sum = ~sum & 0xffff;
1229 DTRACE_MPTCP2(checksum__result, struct mbuf *, m, uint32_t, sum);
1230
1231 return (uint16_t)sum;
1232 }
1233
1234 /*
1235 * When WiFi signal starts fading, there's more loss and RTT spikes.
1236 * Check if there has been a large spike by comparing against
1237 * a tolerable RTT spike threshold.
1238 */
1239 boolean_t
mptcp_no_rto_spike(struct socket * so)1240 mptcp_no_rto_spike(struct socket *so)
1241 {
1242 struct tcpcb *tp = intotcpcb(sotoinpcb(so));
1243 int32_t spike = 0;
1244
1245 if (tp->t_rxtcur > mptcp_rtothresh) {
1246 spike = tp->t_rxtcur - mptcp_rtothresh;
1247 }
1248
1249 if (spike > 0) {
1250 return FALSE;
1251 } else {
1252 return TRUE;
1253 }
1254 }
1255
1256 void
mptcp_handle_deferred_upcalls(struct mppcb * mpp,uint32_t flag)1257 mptcp_handle_deferred_upcalls(struct mppcb *mpp, uint32_t flag)
1258 {
1259 VERIFY(mpp->mpp_flags & flag);
1260 mpp->mpp_flags &= ~flag;
1261
1262 if (mptcp_should_defer_upcall(mpp)) {
1263 return;
1264 }
1265
1266 if (mpp->mpp_flags & MPP_SHOULD_WORKLOOP) {
1267 mpp->mpp_flags &= ~MPP_SHOULD_WORKLOOP;
1268
1269 mptcp_subflow_workloop(mpp->mpp_pcbe);
1270 }
1271
1272 if (mpp->mpp_flags & MPP_SHOULD_RWAKEUP) {
1273 mpp->mpp_flags &= ~MPP_SHOULD_RWAKEUP;
1274
1275 sorwakeup(mpp->mpp_socket);
1276 }
1277
1278 if (mpp->mpp_flags & MPP_SHOULD_WWAKEUP) {
1279 mpp->mpp_flags &= ~MPP_SHOULD_WWAKEUP;
1280
1281 sowwakeup(mpp->mpp_socket);
1282 }
1283 }
1284
1285 static void
mptcp_reset_itfinfo(struct mpt_itf_info * info)1286 mptcp_reset_itfinfo(struct mpt_itf_info *info)
1287 {
1288 memset(info, 0, sizeof(*info));
1289 }
1290
1291 void
mptcp_session_necp_cb(void * handle,int action,uint32_t interface_index,uint32_t necp_flags,__unused bool * viable)1292 mptcp_session_necp_cb(void *handle, int action, uint32_t interface_index,
1293 uint32_t necp_flags, __unused bool *viable)
1294 {
1295 boolean_t has_v4 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV4);
1296 boolean_t has_v6 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_IPV6);
1297 boolean_t has_nat64 = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_HAS_NAT64);
1298 boolean_t low_power = !!(necp_flags & NECP_CLIENT_RESULT_FLAG_INTERFACE_LOW_POWER);
1299 struct mppcb *mp = (struct mppcb *)handle;
1300 struct mptses *mpte = mptompte(mp);
1301 struct socket *mp_so;
1302 struct mptcb *mp_tp;
1303 uint32_t i, ifindex;
1304 struct ifnet *ifp;
1305 int locked = 0;
1306
1307 ifindex = interface_index;
1308 VERIFY(ifindex != IFSCOPE_NONE);
1309
1310 /* About to be garbage-collected (see note about MPTCP/NECP interactions) */
1311 if (mp->mpp_socket->so_usecount == 0) {
1312 return;
1313 }
1314
1315 mp_so = mptetoso(mpte);
1316
1317 if (action != NECP_CLIENT_CBACTION_INITIAL) {
1318 socket_lock(mp_so, 1);
1319 locked = 1;
1320
1321 /* Check again, because it might have changed while waiting */
1322 if (mp->mpp_socket->so_usecount == 0) {
1323 goto out;
1324 }
1325 }
1326
1327 socket_lock_assert_owned(mp_so);
1328
1329 mp_tp = mpte->mpte_mptcb;
1330
1331 ifnet_head_lock_shared();
1332 ifp = ifindex2ifnet[ifindex];
1333 ifnet_head_done();
1334
1335 os_log(mptcp_log_handle, "%s - %lx: action: %u ifindex %u delegated to %u usecount %u mpt_flags %#x state %u v4 %u v6 %u nat64 %u power %u\n",
1336 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mpte), action, ifindex,
1337 ifp && ifp->if_delegated.ifp ? ifp->if_delegated.ifp->if_index : IFSCOPE_NONE,
1338 mp->mpp_socket->so_usecount, mp_tp->mpt_flags, mp_tp->mpt_state,
1339 has_v4, has_v6, has_nat64, low_power);
1340
1341 /* No need on fallen back sockets */
1342 if (mp_tp->mpt_flags & MPTCPF_FALLBACK_TO_TCP) {
1343 goto out;
1344 }
1345
1346 /*
1347 * When the interface goes in low-power mode we don't want to establish
1348 * new subflows on it. Thus, mark it internally as non-viable.
1349 */
1350 if (low_power) {
1351 action = NECP_CLIENT_CBACTION_NONVIABLE;
1352 }
1353
1354 if (action == NECP_CLIENT_CBACTION_INITIAL) {
1355 mpte->mpte_flags |= MPTE_ITFINFO_INIT;
1356 }
1357
1358 if (action == NECP_CLIENT_CBACTION_NONVIABLE) {
1359 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1360 if (mpte->mpte_itfinfo[i].ifindex == IFSCOPE_NONE) {
1361 continue;
1362 }
1363
1364 if (mpte->mpte_itfinfo[i].ifindex == ifindex) {
1365 mptcp_reset_itfinfo(&mpte->mpte_itfinfo[i]);
1366 }
1367 }
1368
1369 mptcp_sched_create_subflows(mpte);
1370 } else if (action == NECP_CLIENT_CBACTION_VIABLE ||
1371 action == NECP_CLIENT_CBACTION_INITIAL) {
1372 int found_slot = 0, slot_index = -1;
1373 struct sockaddr *dst;
1374
1375 if (ifp == NULL) {
1376 goto out;
1377 }
1378
1379 if (IFNET_IS_COMPANION_LINK(ifp)) {
1380 goto out;
1381 }
1382
1383 if (IFNET_IS_EXPENSIVE(ifp) &&
1384 (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) {
1385 goto out;
1386 }
1387
1388 if (IFNET_IS_CONSTRAINED(ifp) &&
1389 (mp_so->so_restrictions & SO_RESTRICT_DENY_CONSTRAINED)) {
1390 goto out;
1391 }
1392
1393 if (IFNET_IS_CELLULAR(ifp) &&
1394 (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) {
1395 goto out;
1396 }
1397
1398 if (IS_INTF_CLAT46(ifp)) {
1399 has_v4 = FALSE;
1400 }
1401
1402 /* Look for the slot on where to store/update the interface-info. */
1403 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1404 /* Found a potential empty slot where we can put it */
1405 if (mpte->mpte_itfinfo[i].ifindex == 0) {
1406 found_slot = 1;
1407 slot_index = i;
1408 }
1409
1410 /*
1411 * The interface is already in our array. Check if we
1412 * need to update it.
1413 */
1414 if (mpte->mpte_itfinfo[i].ifindex == ifindex &&
1415 (mpte->mpte_itfinfo[i].has_v4_conn != has_v4 ||
1416 mpte->mpte_itfinfo[i].has_v6_conn != has_v6 ||
1417 mpte->mpte_itfinfo[i].has_nat64_conn != has_nat64)) {
1418 found_slot = 1;
1419 slot_index = i;
1420 break;
1421 }
1422
1423 if (mpte->mpte_itfinfo[i].ifindex == ifindex) {
1424 /*
1425 * Ok, it's already there and we don't need
1426 * to update it
1427 */
1428 goto out;
1429 }
1430 }
1431
1432 dst = mptcp_get_session_dst(mpte, has_v6, has_v4);
1433 if (dst && dst->sa_family == AF_INET &&
1434 has_v6 && !has_nat64 && !has_v4) {
1435 if (found_slot) {
1436 mpte->mpte_itfinfo[slot_index].ifindex = ifindex;
1437 mpte->mpte_itfinfo[slot_index].has_v4_conn = has_v4;
1438 mpte->mpte_itfinfo[slot_index].has_v6_conn = has_v6;
1439 mpte->mpte_itfinfo[slot_index].has_nat64_conn = has_nat64;
1440 }
1441 goto out;
1442 }
1443
1444 if (found_slot == 0) {
1445 int new_size = mpte->mpte_itfinfo_size * 2;
1446 struct mpt_itf_info *info = kalloc_data(sizeof(*info) * new_size, Z_ZERO);
1447
1448 if (info == NULL) {
1449 os_log_error(mptcp_log_handle, "%s - %lx: malloc failed for %u\n",
1450 __func__, (unsigned long)VM_KERNEL_ADDRPERM(mpte), new_size);
1451 goto out;
1452 }
1453
1454 memcpy(info, mpte->mpte_itfinfo, mpte->mpte_itfinfo_size * sizeof(*info));
1455
1456 if (mpte->mpte_itfinfo_size > MPTE_ITFINFO_SIZE) {
1457 kfree_data(mpte->mpte_itfinfo,
1458 sizeof(*info) * mpte->mpte_itfinfo_size);
1459 }
1460
1461 /* We allocated a new one, thus the first must be empty */
1462 slot_index = mpte->mpte_itfinfo_size;
1463
1464 mpte->mpte_itfinfo = info;
1465 mpte->mpte_itfinfo_size = new_size;
1466 }
1467
1468 VERIFY(slot_index >= 0 && slot_index < (int)mpte->mpte_itfinfo_size);
1469 mpte->mpte_itfinfo[slot_index].ifindex = ifindex;
1470 mpte->mpte_itfinfo[slot_index].has_v4_conn = has_v4;
1471 mpte->mpte_itfinfo[slot_index].has_v6_conn = has_v6;
1472 mpte->mpte_itfinfo[slot_index].has_nat64_conn = has_nat64;
1473
1474 mptcp_sched_create_subflows(mpte);
1475 }
1476
1477 out:
1478 if (locked) {
1479 socket_unlock(mp_so, 1);
1480 }
1481 }
1482
1483 void
mptcp_set_restrictions(struct socket * mp_so)1484 mptcp_set_restrictions(struct socket *mp_so)
1485 {
1486 struct mptses *mpte = mpsotompte(mp_so);
1487 uint32_t i;
1488
1489 socket_lock_assert_owned(mp_so);
1490
1491 ifnet_head_lock_shared();
1492
1493 for (i = 0; i < mpte->mpte_itfinfo_size; i++) {
1494 struct mpt_itf_info *info = &mpte->mpte_itfinfo[i];
1495 uint32_t ifindex = info->ifindex;
1496 struct ifnet *ifp;
1497
1498 if (ifindex == IFSCOPE_NONE) {
1499 continue;
1500 }
1501
1502 ifp = ifindex2ifnet[ifindex];
1503 if (ifp == NULL) {
1504 continue;
1505 }
1506
1507 if (IFNET_IS_EXPENSIVE(ifp) &&
1508 (mp_so->so_restrictions & SO_RESTRICT_DENY_EXPENSIVE)) {
1509 info->ifindex = IFSCOPE_NONE;
1510 }
1511
1512 if (IFNET_IS_CONSTRAINED(ifp) &&
1513 (mp_so->so_restrictions & SO_RESTRICT_DENY_CONSTRAINED)) {
1514 info->ifindex = IFSCOPE_NONE;
1515 }
1516
1517 if (IFNET_IS_CELLULAR(ifp) &&
1518 (mp_so->so_restrictions & SO_RESTRICT_DENY_CELLULAR)) {
1519 info->ifindex = IFSCOPE_NONE;
1520 }
1521 }
1522
1523 ifnet_head_done();
1524 }
1525
1526 #define DUMP_BUF_CHK() { \
1527 clen -= k; \
1528 if (clen < 1) \
1529 goto done; \
1530 c += k; \
1531 }
1532
1533 int
dump_mptcp_reass_qlen(char * str,int str_len)1534 dump_mptcp_reass_qlen(char *str, int str_len)
1535 {
1536 char *c = str;
1537 int k, clen = str_len;
1538
1539 if (mptcp_reass_total_qlen != 0) {
1540 k = scnprintf(c, clen, "\nmptcp reass qlen %d\n", mptcp_reass_total_qlen);
1541 DUMP_BUF_CHK();
1542 }
1543
1544 done:
1545 return str_len - clen;
1546 }
1547