1 /*
2 * Copyright (c) 2000-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_timer.c 8.2 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_timer.c,v 1.34.2.11 2001/08/22 00:59:12 silby Exp $
62 */
63
64 #include "tcp_includes.h"
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/mbuf.h>
70 #include <sys/sysctl.h>
71 #include <sys/socket.h>
72 #include <sys/socketvar.h>
73 #include <sys/protosw.h>
74 #include <sys/domain.h>
75 #include <sys/mcache.h>
76 #include <sys/queue.h>
77 #include <kern/locks.h>
78 #include <kern/cpu_number.h> /* before tcp_seq.h, for tcp_random18() */
79 #include <mach/boolean.h>
80
81 #include <net/route.h>
82 #include <net/if_var.h>
83 #include <net/ntstat.h>
84
85 #include <netinet/in.h>
86 #include <netinet/in_systm.h>
87 #include <netinet/in_pcb.h>
88 #include <netinet/in_var.h>
89 #include <netinet6/in6_pcb.h>
90 #include <netinet/ip_var.h>
91 #include <netinet/tcp.h>
92 #include <netinet/tcp_cache.h>
93 #include <netinet/tcp_fsm.h>
94 #include <netinet/tcp_seq.h>
95 #include <netinet/tcp_timer.h>
96 #include <netinet/tcp_var.h>
97 #include <netinet/tcp_cc.h>
98 #include <netinet6/tcp6_var.h>
99 #include <netinet/tcpip.h>
100 #include <netinet/tcp_log.h>
101
102 #include <sys/kdebug.h>
103 #include <mach/sdt.h>
104 #include <netinet/mptcp_var.h>
105 #include <net/content_filter.h>
106 #include <net/sockaddr_utils.h>
107
108 /* Max number of times a stretch ack can be delayed on a connection */
109 #define TCP_STRETCHACK_DELAY_THRESHOLD 5
110
111 /*
112 * If the host processor has been sleeping for too long, this is the threshold
113 * used to avoid sending stale retransmissions.
114 */
115 #define TCP_SLEEP_TOO_LONG (10 * 60 * 1000) /* 10 minutes in ms */
116
117 /* tcp timer list */
118 struct tcptimerlist tcp_timer_list;
119
120 /* List of pcbs in timewait state, protected by tcbinfo's ipi_lock */
121 struct tcptailq tcp_tw_tailq;
122
123
124 static int
125 sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
126 {
127 #pragma unused(arg2)
128 int error, temp;
129 long s, tt;
130
131 tt = *(int *)arg1;
132 s = tt * 1000 / TCP_RETRANSHZ;
133 if (tt < 0 || s > INT_MAX) {
134 return EINVAL;
135 }
136 temp = (int)s;
137
138 error = sysctl_handle_int(oidp, &temp, 0, req);
139 if (error || !req->newptr) {
140 return error;
141 }
142
143 tt = (long)temp * TCP_RETRANSHZ / 1000;
144 if (tt < 1 || tt > INT_MAX) {
145 return EINVAL;
146 }
147
148 *(int *)arg1 = (int)tt;
149 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1);
150 return 0;
151 }
152
153 #if SYSCTL_SKMEM
154 int tcp_keepinit = TCPTV_KEEP_INIT;
155 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
156 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
157 &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit),
158 sysctl_msec_to_ticks, "I", "");
159
160 int tcp_keepidle = TCPTV_KEEP_IDLE;
161 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
162 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
163 &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle),
164 sysctl_msec_to_ticks, "I", "");
165
166 int tcp_keepintvl = TCPTV_KEEPINTVL;
167 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
168 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
169 &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl),
170 sysctl_msec_to_ticks, "I", "");
171
172 SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt,
173 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
174 int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive");
175
176 int tcp_msl = TCPTV_MSL;
177 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
178 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
179 &tcp_msl, offsetof(skmem_sysctl, tcp.msl),
180 sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
181 #else /* SYSCTL_SKMEM */
182 int tcp_keepinit;
183 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
184 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
185 &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", "");
186
187 int tcp_keepidle;
188 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
189 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
190 &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", "");
191
192 int tcp_keepintvl;
193 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
194 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
195 &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
196
197 int tcp_keepcnt;
198 SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt,
199 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
200 &tcp_keepcnt, 0, "number of times to repeat keepalive");
201
202 int tcp_msl;
203 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
204 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
205 &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
206 #endif /* SYSCTL_SKMEM */
207
208 /*
209 * Avoid DoS with connections half-closed in TIME_WAIT_2
210 */
211 int tcp_fin_timeout = TCPTV_FINWAIT2;
212
213 static int
214 sysctl_tcp_fin_timeout SYSCTL_HANDLER_ARGS
215 {
216 #pragma unused(arg2)
217 int error;
218 int value = tcp_fin_timeout;
219
220 error = sysctl_handle_int(oidp, &value, 0, req);
221 if (error != 0 || req->newptr == USER_ADDR_NULL) {
222 return error;
223 }
224
225 if (value == -1) {
226 /* Reset to default value */
227 value = TCPTV_FINWAIT2;
228 } else {
229 /* Convert from milliseconds */
230 long big_value = value * TCP_RETRANSHZ / 1000;
231
232 if (big_value < 0 || big_value > INT_MAX) {
233 return EINVAL;
234 }
235 value = (int)big_value;
236 }
237 tcp_fin_timeout = value;
238 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, value);
239 return 0;
240 }
241
242 #if SYSCTL_SKMEM
243 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
244 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
245 &tcp_fin_timeout, offsetof(skmem_sysctl, tcp.fin_timeout),
246 sysctl_tcp_fin_timeout, "I", "");
247 #else /* SYSCTL_SKMEM */
248 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
249 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
250 &tcp_fin_timeout, 0,
251 sysctl_tcp_fin_timeout, "I", "");
252 #endif /* SYSCTL_SKMEM */
253
254 /*
255 * Avoid DoS via TCP Robustness in Persist Condition
256 * (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt)
257 * by allowing a system wide maximum persistence timeout value when in
258 * Zero Window Probe mode.
259 *
260 * Expressed in milliseconds to be consistent without timeout related
261 * values, the TCP socket option is in seconds.
262 */
263 #if SYSCTL_SKMEM
264 u_int32_t tcp_max_persist_timeout = 0;
265 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
266 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
267 &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout),
268 sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP");
269 #else /* SYSCTL_SKMEM */
270 u_int32_t tcp_max_persist_timeout = 0;
271 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
272 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
273 &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I",
274 "Maximum persistence timeout for ZWP");
275 #endif /* SYSCTL_SKMEM */
276
277 SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive,
278 CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0,
279 "Assume SO_KEEPALIVE on all TCP connections");
280
281 /*
282 * This parameter determines how long the timer list will stay in fast or
283 * quick mode even though all connections are idle. In this state, the
284 * timer will run more frequently anticipating new data.
285 */
286 SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax,
287 CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax,
288 TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode");
289
290 /*
291 * See tcp_syn_backoff[] for interval values between SYN retransmits;
292 * the value set below defines the number of retransmits, before we
293 * disable the timestamp and window scaling options during subsequent
294 * SYN retransmits. Setting it to 0 disables the dropping off of those
295 * two options.
296 */
297 SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres,
298 CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres,
299 10, "Number of retransmitted SYNs before disabling RFC 1323 "
300 "options on local connections");
301
302 static int tcp_timer_advanced = 0;
303 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced,
304 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_timer_advanced, 0,
305 "Number of times one of the timers was advanced");
306
307 static int tcp_resched_timerlist = 0;
308 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist,
309 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0,
310 "Number of times timer list was rescheduled as part of processing a packet");
311
312 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection,
313 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1,
314 "Path MTU Discovery Black Hole Detection");
315
316 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss,
317 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200,
318 "Path MTU Discovery Black Hole Detection lowered MSS");
319
320 #if (DEBUG || DEVELOPMENT)
321 int tcp_probe_if_fix_port = 0;
322 SYSCTL_INT(_net_inet_tcp, OID_AUTO, probe_if_fix_port,
323 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
324 &tcp_probe_if_fix_port, 0, "");
325 #endif /* (DEBUG || DEVELOPMENT) */
326
327 static u_int32_t tcp_mss_rec_medium = 1200;
328 static u_int32_t tcp_mss_rec_low = 512;
329
330 #define TCP_REPORT_STATS_INTERVAL 43200 /* 12 hours, in seconds */
331 int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL;
332
333 /* performed garbage collection of "used" sockets */
334 static boolean_t tcp_gc_done = FALSE;
335
336 /* max idle probes */
337 int tcp_maxpersistidle = TCPTV_KEEP_IDLE;
338
339 /*
340 * TCP delack timer is set to 100 ms. Since the processing of timer list
341 * in fast mode will happen no faster than 100 ms, the delayed ack timer
342 * will fire some where between 100 and 200 ms.
343 */
344 int tcp_delack = TCP_RETRANSHZ / 10;
345
346 #if MPTCP
347 /*
348 * MP_JOIN retransmission of 3rd ACK will be every 500 msecs without backoff
349 */
350 int tcp_jack_rxmt = TCP_RETRANSHZ / 2;
351 #endif /* MPTCP */
352
353 static boolean_t tcp_itimer_done = FALSE;
354
355 static void tcp_remove_timer(struct tcpcb *tp);
356 static void tcp_sched_timerlist(uint32_t offset);
357 static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode,
358 u_int16_t probe_if_index);
359 static inline void tcp_set_lotimer_index(struct tcpcb *);
360 __private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp);
361 static inline void tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp);
362 __private_extern__ void tcp_report_stats(void);
363
364 static u_int64_t tcp_last_report_time;
365
366 /*
367 * Structure to store previously reported stats so that we can send
368 * incremental changes in each report interval.
369 */
370 struct tcp_last_report_stats {
371 u_int32_t tcps_connattempt;
372 u_int32_t tcps_accepts;
373 u_int32_t tcps_ecn_client_setup;
374 u_int32_t tcps_ecn_server_setup;
375 u_int32_t tcps_ecn_client_success;
376 u_int32_t tcps_ecn_server_success;
377 u_int32_t tcps_ecn_not_supported;
378 u_int32_t tcps_ecn_lost_syn;
379 u_int32_t tcps_ecn_lost_synack;
380 u_int32_t tcps_ecn_recv_ce;
381 u_int32_t tcps_ecn_recv_ece;
382 u_int32_t tcps_ecn_sent_ece;
383 u_int32_t tcps_ecn_conn_recv_ce;
384 u_int32_t tcps_ecn_conn_recv_ece;
385 u_int32_t tcps_ecn_conn_plnoce;
386 u_int32_t tcps_ecn_conn_pl_ce;
387 u_int32_t tcps_ecn_conn_nopl_ce;
388 u_int32_t tcps_ecn_fallback_synloss;
389 u_int32_t tcps_ecn_fallback_reorder;
390 u_int32_t tcps_ecn_fallback_ce;
391
392 /* TFO-related statistics */
393 u_int32_t tcps_tfo_syn_data_rcv;
394 u_int32_t tcps_tfo_cookie_req_rcv;
395 u_int32_t tcps_tfo_cookie_sent;
396 u_int32_t tcps_tfo_cookie_invalid;
397 u_int32_t tcps_tfo_cookie_req;
398 u_int32_t tcps_tfo_cookie_rcv;
399 u_int32_t tcps_tfo_syn_data_sent;
400 u_int32_t tcps_tfo_syn_data_acked;
401 u_int32_t tcps_tfo_syn_loss;
402 u_int32_t tcps_tfo_blackhole;
403 u_int32_t tcps_tfo_cookie_wrong;
404 u_int32_t tcps_tfo_no_cookie_rcv;
405 u_int32_t tcps_tfo_heuristics_disable;
406 u_int32_t tcps_tfo_sndblackhole;
407
408 /* MPTCP-related statistics */
409 u_int32_t tcps_mptcp_handover_attempt;
410 u_int32_t tcps_mptcp_interactive_attempt;
411 u_int32_t tcps_mptcp_aggregate_attempt;
412 u_int32_t tcps_mptcp_fp_handover_attempt;
413 u_int32_t tcps_mptcp_fp_interactive_attempt;
414 u_int32_t tcps_mptcp_fp_aggregate_attempt;
415 u_int32_t tcps_mptcp_heuristic_fallback;
416 u_int32_t tcps_mptcp_fp_heuristic_fallback;
417 u_int32_t tcps_mptcp_handover_success_wifi;
418 u_int32_t tcps_mptcp_handover_success_cell;
419 u_int32_t tcps_mptcp_interactive_success;
420 u_int32_t tcps_mptcp_aggregate_success;
421 u_int32_t tcps_mptcp_fp_handover_success_wifi;
422 u_int32_t tcps_mptcp_fp_handover_success_cell;
423 u_int32_t tcps_mptcp_fp_interactive_success;
424 u_int32_t tcps_mptcp_fp_aggregate_success;
425 u_int32_t tcps_mptcp_handover_cell_from_wifi;
426 u_int32_t tcps_mptcp_handover_wifi_from_cell;
427 u_int32_t tcps_mptcp_interactive_cell_from_wifi;
428 u_int64_t tcps_mptcp_handover_cell_bytes;
429 u_int64_t tcps_mptcp_interactive_cell_bytes;
430 u_int64_t tcps_mptcp_aggregate_cell_bytes;
431 u_int64_t tcps_mptcp_handover_all_bytes;
432 u_int64_t tcps_mptcp_interactive_all_bytes;
433 u_int64_t tcps_mptcp_aggregate_all_bytes;
434 u_int32_t tcps_mptcp_back_to_wifi;
435 u_int32_t tcps_mptcp_wifi_proxy;
436 u_int32_t tcps_mptcp_cell_proxy;
437 u_int32_t tcps_mptcp_triggered_cell;
438 };
439
440
441 /* Returns true if the timer is on the timer list */
442 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
443
444 /* Run the TCP timerlist atleast once every hour */
445 #define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ)
446
447
448 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
449 static boolean_t tcp_garbage_collect(struct inpcb *, int);
450
451 #define TIMERENTRY_TO_TP(te) (__unsafe_forge_single(struct tcpcb *, ((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next))))
452
453 #define VERIFY_NEXT_LINK(elm, field) do { \
454 if (LIST_NEXT((elm),field) != NULL && \
455 LIST_NEXT((elm),field)->field.le_prev != \
456 &((elm)->field.le_next)) \
457 panic("Bad link elm %p next->prev != elm", (elm)); \
458 } while(0)
459
460 #define VERIFY_PREV_LINK(elm, field) do { \
461 if (*(elm)->field.le_prev != (elm)) \
462 panic("Bad link elm %p prev->next != elm", (elm)); \
463 } while(0)
464
465 #define TCP_SET_TIMER_MODE(mode, i) do { \
466 if (IS_TIMER_HZ_10MS(i)) \
467 (mode) |= TCP_TIMERLIST_10MS_MODE; \
468 else if (IS_TIMER_HZ_100MS(i)) \
469 (mode) |= TCP_TIMERLIST_100MS_MODE; \
470 else \
471 (mode) |= TCP_TIMERLIST_500MS_MODE; \
472 } while(0)
473
474 #if (DEVELOPMENT || DEBUG)
475 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_medium,
476 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_medium, 0,
477 "Medium MSS based on recommendation in link status report");
478 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_low,
479 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_low, 0,
480 "Low MSS based on recommendation in link status report");
481
482 static int32_t tcp_change_mss_recommended = 0;
483 static int
484 sysctl_change_mss_recommended SYSCTL_HANDLER_ARGS
485 {
486 #pragma unused(oidp, arg1, arg2)
487 int i, err = 0, changed = 0;
488 struct ifnet *ifp;
489 struct if_link_status ifsr;
490 struct if_cellular_status_v1 *new_cell_sr;
491 err = sysctl_io_number(req, tcp_change_mss_recommended,
492 sizeof(int32_t), &i, &changed);
493 if (changed) {
494 if (i < 0 || i > UINT16_MAX) {
495 return EINVAL;
496 }
497 ifnet_head_lock_shared();
498 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
499 if (IFNET_IS_CELLULAR(ifp)) {
500 bzero(&ifsr, sizeof(ifsr));
501 new_cell_sr = &ifsr.ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
502 ifsr.ifsr_version = IF_CELLULAR_STATUS_REPORT_CURRENT_VERSION;
503 ifsr.ifsr_len = sizeof(*new_cell_sr);
504
505 /* Set MSS recommended */
506 new_cell_sr->valid_bitmask |= IF_CELL_UL_MSS_RECOMMENDED_VALID;
507 new_cell_sr->mss_recommended = (uint16_t)i;
508 err = ifnet_link_status_report(ifp, new_cell_sr, sizeof(new_cell_sr));
509 if (err == 0) {
510 tcp_change_mss_recommended = i;
511 } else {
512 break;
513 }
514 }
515 }
516 ifnet_head_done();
517 }
518 return err;
519 }
520
521 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, change_mss_recommended,
522 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_change_mss_recommended,
523 0, sysctl_change_mss_recommended, "IU", "Change MSS recommended");
524
525 SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval,
526 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0,
527 "Report stats interval");
528 #endif /* (DEVELOPMENT || DEBUG) */
529
530 /*
531 * Macro to compare two timers. If there is a reset of the sign bit,
532 * it is safe to assume that the timer has wrapped around. By doing
533 * signed comparision, we take care of wrap around such that the value
534 * with the sign bit reset is actually ahead of the other.
535 */
536 inline int32_t
timer_diff(uint32_t t1,uint32_t toff1,uint32_t t2,uint32_t toff2)537 timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2)
538 {
539 return (int32_t)((t1 + toff1) - (t2 + toff2));
540 }
541
542 /*
543 * Add to tcp timewait list, delay is given in milliseconds.
544 */
545 static void
add_to_time_wait_locked(struct tcpcb * tp,uint32_t delay)546 add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay)
547 {
548 struct inpcbinfo *pcbinfo = &tcbinfo;
549 struct inpcb *inp = tp->t_inpcb;
550 uint32_t timer;
551
552 /* pcb list should be locked when we get here */
553 LCK_RW_ASSERT(&pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE);
554
555 /* We may get here multiple times, so check */
556 if (!(inp->inp_flags2 & INP2_TIMEWAIT)) {
557 pcbinfo->ipi_twcount++;
558 inp->inp_flags2 |= INP2_TIMEWAIT;
559
560 /* Remove from global inp list */
561 LIST_REMOVE(inp, inp_list);
562 } else {
563 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
564 }
565
566 /* Compute the time at which this socket can be closed */
567 timer = tcp_now + delay;
568
569 /* We will use the TCPT_2MSL timer for tracking this delay */
570
571 if (TIMER_IS_ON_LIST(tp)) {
572 tcp_remove_timer(tp);
573 }
574 tp->t_timer[TCPT_2MSL] = timer;
575
576 TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry);
577 }
578
579 void
add_to_time_wait(struct tcpcb * tp,uint32_t delay)580 add_to_time_wait(struct tcpcb *tp, uint32_t delay)
581 {
582 if (tp->t_inpcb->inp_socket->so_options & SO_NOWAKEFROMSLEEP) {
583 socket_post_kev_msg_closed(tp->t_inpcb->inp_socket);
584 }
585
586 tcp_del_fsw_flow(tp);
587
588 /* 19182803: Notify nstat that connection is closing before waiting. */
589 nstat_pcb_detach(tp->t_inpcb);
590
591 #if CONTENT_FILTER
592 if ((tp->t_inpcb->inp_socket->so_flags & SOF_CONTENT_FILTER) != 0) {
593 /* If filter present, allow filter to finish processing all queued up data before adding to time wait queue */
594 (void) cfil_sock_tcp_add_time_wait(tp->t_inpcb->inp_socket);
595 } else
596 #endif /* CONTENT_FILTER */
597 {
598 add_to_time_wait_now(tp, delay);
599 }
600 }
601
602 void
add_to_time_wait_now(struct tcpcb * tp,uint32_t delay)603 add_to_time_wait_now(struct tcpcb *tp, uint32_t delay)
604 {
605 struct inpcbinfo *pcbinfo = &tcbinfo;
606
607 if (!lck_rw_try_lock_exclusive(&pcbinfo->ipi_lock)) {
608 socket_unlock(tp->t_inpcb->inp_socket, 0);
609 lck_rw_lock_exclusive(&pcbinfo->ipi_lock);
610 socket_lock(tp->t_inpcb->inp_socket, 0);
611 }
612 add_to_time_wait_locked(tp, delay);
613 lck_rw_done(&pcbinfo->ipi_lock);
614
615 inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY);
616 }
617
618 /* If this is on time wait queue, remove it. */
619 void
tcp_remove_from_time_wait(struct inpcb * inp)620 tcp_remove_from_time_wait(struct inpcb *inp)
621 {
622 struct tcpcb *tp = intotcpcb(inp);
623 if (inp->inp_flags2 & INP2_TIMEWAIT) {
624 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
625 }
626 }
627
628 static boolean_t
tcp_garbage_collect(struct inpcb * inp,int istimewait)629 tcp_garbage_collect(struct inpcb *inp, int istimewait)
630 {
631 boolean_t active = FALSE;
632 struct socket *so, *mp_so = NULL;
633 struct tcpcb *tp;
634
635 so = inp->inp_socket;
636 tp = intotcpcb(inp);
637
638 if (so->so_flags & SOF_MP_SUBFLOW) {
639 mp_so = mptetoso(tptomptp(tp)->mpt_mpte);
640 if (!socket_try_lock(mp_so)) {
641 mp_so = NULL;
642 active = TRUE;
643 goto out;
644 }
645 if (mpsotomppcb(mp_so)->mpp_inside > 0) {
646 os_log(mptcp_log_handle, "%s - %lx: Still inside %d usecount %d\n", __func__,
647 (unsigned long)VM_KERNEL_ADDRPERM(mpsotompte(mp_so)),
648 mpsotomppcb(mp_so)->mpp_inside,
649 mp_so->so_usecount);
650 socket_unlock(mp_so, 0);
651 mp_so = NULL;
652 active = TRUE;
653 goto out;
654 }
655 /* We call socket_unlock with refcount further below */
656 mp_so->so_usecount++;
657 tptomptp(tp)->mpt_mpte->mpte_mppcb->mpp_inside++;
658 }
659
660 /*
661 * Skip if still in use or busy; it would have been more efficient
662 * if we were to test so_usecount against 0, but this isn't possible
663 * due to the current implementation of tcp_dropdropablreq() where
664 * overflow sockets that are eligible for garbage collection have
665 * their usecounts set to 1.
666 */
667 if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) {
668 active = TRUE;
669 goto out;
670 }
671
672 /* Check again under the lock */
673 if (so->so_usecount > 1) {
674 if (inp->inp_wantcnt == WNT_STOPUSING) {
675 active = TRUE;
676 }
677 lck_mtx_unlock(&inp->inpcb_mtx);
678 goto out;
679 }
680
681 if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) &&
682 tp->t_state != TCPS_CLOSED) {
683 /* Become a regular mutex */
684 lck_mtx_convert_spin(&inp->inpcb_mtx);
685 tcp_close(tp);
686 }
687
688 /*
689 * Overflowed socket dropped from the listening queue? Do this
690 * only if we are called to clean up the time wait slots, since
691 * tcp_dropdropablreq() considers a socket to have been fully
692 * dropped after add_to_time_wait() is finished.
693 * Also handle the case of connections getting closed by the peer
694 * while in the queue as seen with rdar://6422317
695 *
696 */
697 if (so->so_usecount == 1 &&
698 ((istimewait && (so->so_flags & SOF_OVERFLOW)) ||
699 ((tp != NULL) && (tp->t_state == TCPS_CLOSED) &&
700 (so->so_head != NULL) &&
701 ((so->so_state & (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE)) ==
702 (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE))))) {
703 if (inp->inp_state != INPCB_STATE_DEAD) {
704 /* Become a regular mutex */
705 lck_mtx_convert_spin(&inp->inpcb_mtx);
706 if (SOCK_CHECK_DOM(so, PF_INET6)) {
707 in6_pcbdetach(inp);
708 } else {
709 in_pcbdetach(inp);
710 }
711 }
712 VERIFY(so->so_usecount > 0);
713 so->so_usecount--;
714 if (inp->inp_wantcnt == WNT_STOPUSING) {
715 active = TRUE;
716 }
717 lck_mtx_unlock(&inp->inpcb_mtx);
718 goto out;
719 } else if (inp->inp_wantcnt != WNT_STOPUSING) {
720 lck_mtx_unlock(&inp->inpcb_mtx);
721 active = FALSE;
722 goto out;
723 }
724
725 /*
726 * We get here because the PCB is no longer searchable
727 * (WNT_STOPUSING); detach (if needed) and dispose if it is dead
728 * (usecount is 0). This covers all cases, including overflow
729 * sockets and those that are considered as "embryonic",
730 * i.e. created by sonewconn() in TCP input path, and have
731 * not yet been committed. For the former, we reduce the usecount
732 * to 0 as done by the code above. For the latter, the usecount
733 * would have reduced to 0 as part calling soabort() when the
734 * socket is dropped at the end of tcp_input().
735 */
736 if (so->so_usecount == 0) {
737 DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp,
738 struct tcpcb *, tp, int32_t, TCPS_CLOSED);
739 /* Become a regular mutex */
740 lck_mtx_convert_spin(&inp->inpcb_mtx);
741
742 /*
743 * If this tp still happens to be on the timer list,
744 * take it out
745 */
746 if (TIMER_IS_ON_LIST(tp)) {
747 tcp_remove_timer(tp);
748 }
749
750 if (inp->inp_state != INPCB_STATE_DEAD) {
751 if (SOCK_CHECK_DOM(so, PF_INET6)) {
752 in6_pcbdetach(inp);
753 } else {
754 in_pcbdetach(inp);
755 }
756 }
757
758 if (mp_so) {
759 mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub);
760
761 /* so is now unlinked from mp_so - let's drop the lock */
762 socket_unlock(mp_so, 1);
763 mp_so = NULL;
764 }
765
766 in_pcbdispose(inp);
767 active = FALSE;
768 goto out;
769 }
770
771 lck_mtx_unlock(&inp->inpcb_mtx);
772 active = TRUE;
773
774 out:
775 if (mp_so) {
776 socket_unlock(mp_so, 1);
777 }
778
779 return active;
780 }
781
782 /*
783 * TCP garbage collector callback (inpcb_timer_func_t).
784 *
785 * Returns the number of pcbs that will need to be gc-ed soon,
786 * returnining > 0 will keep timer active.
787 */
788 void
tcp_gc(struct inpcbinfo * ipi)789 tcp_gc(struct inpcbinfo *ipi)
790 {
791 struct inpcb *inp, *nxt;
792 struct tcpcb *tw_tp, *tw_ntp;
793 #if KDEBUG
794 static int tws_checked = 0;
795 #endif
796
797 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0);
798
799 /*
800 * Update tcp_now here as it may get used while
801 * processing the slow timer.
802 */
803 calculate_tcp_clock();
804
805 /*
806 * Garbage collect socket/tcpcb: We need to acquire the list lock
807 * exclusively to do this
808 */
809
810 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
811 /* don't sweat it this time; cleanup was done last time */
812 if (tcp_gc_done == TRUE) {
813 tcp_gc_done = FALSE;
814 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END,
815 tws_checked, cur_tw_slot, 0, 0, 0);
816 /* Lock upgrade failed, give up this round */
817 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
818 return;
819 }
820 /* Upgrade failed, lost lock now take it again exclusive */
821 lck_rw_lock_exclusive(&ipi->ipi_lock);
822 }
823 tcp_gc_done = TRUE;
824
825 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
826 if (tcp_garbage_collect(inp, 0)) {
827 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
828 }
829 }
830
831 /* Now cleanup the time wait ones */
832 TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) {
833 /*
834 * We check the timestamp here without holding the
835 * socket lock for better performance. If there are
836 * any pcbs in time-wait, the timer will get rescheduled.
837 * Hence some error in this check can be tolerated.
838 *
839 * Sometimes a socket on time-wait queue can be closed if
840 * 2MSL timer expired but the application still has a
841 * usecount on it.
842 */
843 if (tw_tp->t_state == TCPS_CLOSED ||
844 TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) {
845 if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) {
846 os_atomic_inc(&ipi->ipi_gc_req.intimer_lazy, relaxed);
847 }
848 }
849 }
850
851 /* take into account pcbs that are still in time_wait_slots */
852 os_atomic_add(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount, relaxed);
853
854 lck_rw_done(&ipi->ipi_lock);
855
856 /* Clean up the socache while we are here */
857 if (so_cache_timer()) {
858 os_atomic_inc(&ipi->ipi_gc_req.intimer_lazy, relaxed);
859 }
860
861 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked,
862 cur_tw_slot, 0, 0, 0);
863
864 return;
865 }
866
867 /*
868 * Cancel all timers for TCP tp.
869 */
870 void
tcp_canceltimers(struct tcpcb * tp)871 tcp_canceltimers(struct tcpcb *tp)
872 {
873 int i;
874
875 tcp_remove_timer(tp);
876 for (i = 0; i < TCPT_NTIMERS; i++) {
877 tp->t_timer[i] = 0;
878 }
879 tp->tentry.timer_start = tcp_now;
880 tp->tentry.index = TCPT_NONE;
881 }
882
883 int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =
884 { 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 };
885
886 int tcp_backoff[TCP_MAXRXTSHIFT + 1] =
887 { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
888
889 static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */
890
891 void
tcp_rexmt_save_state(struct tcpcb * tp)892 tcp_rexmt_save_state(struct tcpcb *tp)
893 {
894 u_int32_t fsize;
895 if (TSTMP_SUPPORTED(tp)) {
896 /*
897 * Since timestamps are supported on the connection,
898 * we can do recovery as described in rfc 4015.
899 */
900 fsize = tp->snd_max - tp->snd_una;
901 tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh);
902 tp->snd_recover_prev = tp->snd_recover;
903 } else {
904 /*
905 * Timestamp option is not supported on this connection.
906 * Record ssthresh and cwnd so they can
907 * be recovered if this turns out to be a "bad" retransmit.
908 * A retransmit is considered "bad" if an ACK for this
909 * segment is received within RTT/2 interval; the assumption
910 * here is that the ACK was already in flight. See
911 * "On Estimating End-to-End Network Path Properties" by
912 * Allman and Paxson for more details.
913 */
914 tp->snd_cwnd_prev = tp->snd_cwnd;
915 tp->snd_ssthresh_prev = tp->snd_ssthresh;
916 tp->snd_recover_prev = tp->snd_recover;
917 if (IN_FASTRECOVERY(tp)) {
918 tp->t_flags |= TF_WASFRECOVERY;
919 } else {
920 tp->t_flags &= ~TF_WASFRECOVERY;
921 }
922 }
923 tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2;
924 tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT);
925 tp->t_flagsext &= ~(TF_RECOMPUTE_RTT);
926 }
927
928 /*
929 * Revert to the older segment size if there is an indication that PMTU
930 * blackhole detection was not needed.
931 */
932 void
tcp_pmtud_revert_segment_size(struct tcpcb * tp)933 tcp_pmtud_revert_segment_size(struct tcpcb *tp)
934 {
935 int32_t optlen;
936
937 VERIFY(tp->t_pmtud_saved_maxopd > 0);
938 tp->t_flags |= TF_PMTUD;
939 tp->t_flags &= ~TF_BLACKHOLE;
940 optlen = tp->t_maxopd - tp->t_maxseg;
941 tp->t_maxopd = tp->t_pmtud_saved_maxopd;
942 tp->t_maxseg = tp->t_maxopd - optlen;
943
944 /*
945 * Reset the slow-start flight size as it
946 * may depend on the new MSS
947 */
948 if (CC_ALGO(tp)->cwnd_init != NULL) {
949 CC_ALGO(tp)->cwnd_init(tp);
950 }
951
952 if (TCP_USE_RLEDBAT(tp, tp->t_inpcb->inp_socket) &&
953 tcp_cc_rledbat.rwnd_init != NULL) {
954 tcp_cc_rledbat.rwnd_init(tp);
955 }
956
957 tp->t_pmtud_start_ts = 0;
958 tcpstat.tcps_pmtudbh_reverted++;
959
960 /* change MSS according to recommendation, if there was one */
961 tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL);
962 }
963
964 static uint32_t
tcp_pmtud_black_holed_next_mss(struct tcpcb * tp)965 tcp_pmtud_black_holed_next_mss(struct tcpcb *tp)
966 {
967 /* Reduce the MSS to intermediary value */
968 if (tp->t_maxopd > tcp_pmtud_black_hole_mss) {
969 return tcp_pmtud_black_hole_mss;
970 } else {
971 if (tp->t_inpcb->inp_vflag & INP_IPV4) {
972 return tcp_mssdflt;
973 } else {
974 return tcp_v6mssdflt;
975 }
976 }
977 }
978
979 /*
980 * Send a packet designed to force a response
981 * if the peer is up and reachable:
982 * either an ACK if the connection is still alive,
983 * or an RST if the peer has closed the connection
984 * due to timeout or reboot.
985 * Using sequence number tp->snd_una-1
986 * causes the transmitted zero-length segment
987 * to lie outside the receive window;
988 * by the protocol spec, this requires the
989 * correspondent TCP to respond.
990 */
991 static bool
tcp_send_keep_alive(struct tcpcb * tp)992 tcp_send_keep_alive(struct tcpcb *tp)
993 {
994 struct tcptemp *__single t_template;
995 struct mbuf *__single m;
996
997 tcpstat.tcps_keepprobe++;
998 t_template = tcp_maketemplate(tp, &m);
999 if (t_template != NULL) {
1000 struct inpcb *inp = tp->t_inpcb;
1001 struct tcp_respond_args tra;
1002
1003 bzero(&tra, sizeof(tra));
1004 tra.nocell = INP_NO_CELLULAR(inp) ? 1 : 0;
1005 tra.noexpensive = INP_NO_EXPENSIVE(inp) ? 1 : 0;
1006 tra.noconstrained = INP_NO_CONSTRAINED(inp) ? 1 : 0;
1007 tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp) ? 1 : 0;
1008 tra.intcoproc_allowed = INP_INTCOPROC_ALLOWED(inp) ? 1 : 0;
1009 tra.management_allowed = INP_MANAGEMENT_ALLOWED(inp) ? 1 : 0;
1010 tra.keep_alive = 1;
1011 if (tp->t_inpcb->inp_flags & INP_BOUND_IF) {
1012 tra.ifscope = tp->t_inpcb->inp_boundifp->if_index;
1013 } else {
1014 tra.ifscope = IFSCOPE_NONE;
1015 }
1016 tcp_respond(tp, t_template->tt_ipgen, sizeof(t_template->tt_ipgen),
1017 &t_template->tt_t, (struct mbuf *)NULL,
1018 tp->rcv_nxt, tp->snd_una - 1, 0, &tra);
1019 (void) m_free(m);
1020 return true;
1021 } else {
1022 return false;
1023 }
1024 }
1025
1026 /*
1027 * TCP timer processing.
1028 */
1029 struct tcpcb *
tcp_timers(struct tcpcb * tp,int timer)1030 tcp_timers(struct tcpcb *tp, int timer)
1031 {
1032 int32_t rexmt, optlen = 0, idle_time = 0;
1033 struct socket *so;
1034 u_int64_t accsleep_ms;
1035 u_int64_t last_sleep_ms = 0;
1036 struct ifnet *outifp = tp->t_inpcb->inp_last_outifp;
1037
1038 so = tp->t_inpcb->inp_socket;
1039 idle_time = tcp_now - tp->t_rcvtime;
1040
1041 switch (timer) {
1042 /*
1043 * 2 MSL timeout in shutdown went off. If we're closed but
1044 * still waiting for peer to close and connection has been idle
1045 * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2,
1046 * delete connection control block.
1047 * Otherwise, (this case shouldn't happen) check again in a bit
1048 * we keep the socket in the main list in that case.
1049 */
1050 case TCPT_2MSL:
1051 tcp_free_sackholes(tp);
1052 if (tp->t_state != TCPS_TIME_WAIT &&
1053 tp->t_state != TCPS_FIN_WAIT_2 &&
1054 ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) {
1055 tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp,
1056 (u_int32_t)TCP_CONN_KEEPINTVL(tp));
1057 } else {
1058 if (tp->t_state == TCPS_FIN_WAIT_2) {
1059 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1060 "FIN wait timeout drop");
1061 tcpstat.tcps_fin_timeout_drops++;
1062 tp = tcp_drop(tp, 0);
1063 } else {
1064 tp = tcp_close(tp);
1065 }
1066 return tp;
1067 }
1068 break;
1069
1070 /*
1071 * Retransmission timer went off. Message has not
1072 * been acked within retransmit interval. Back off
1073 * to a longer retransmit interval and retransmit one segment.
1074 */
1075 case TCPT_REXMT:
1076 absolutetime_to_nanoseconds(mach_absolutetime_asleep,
1077 &accsleep_ms);
1078 accsleep_ms = accsleep_ms / 1000000UL;
1079 if (accsleep_ms > tp->t_accsleep_ms) {
1080 last_sleep_ms = accsleep_ms - tp->t_accsleep_ms;
1081 }
1082 /*
1083 * Drop a connection in the retransmit timer
1084 * 1. If we have retransmitted more than TCP_MAXRXTSHIFT
1085 * times
1086 * 2. If the time spent in this retransmission episode is
1087 * more than the time limit set with TCP_RXT_CONNDROPTIME
1088 * socket option
1089 * 3. If TCP_RXT_FINDROP socket option was set and
1090 * we have already retransmitted the FIN 3 times without
1091 * receiving an ack
1092 */
1093 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT ||
1094 (tp->t_rxt_conndroptime > 0 && tp->t_rxtstart > 0 &&
1095 (tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime) ||
1096 ((tp->t_flagsext & TF_RXTFINDROP) != 0 &&
1097 (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) ||
1098 (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) {
1099 if (tp->t_state == TCPS_ESTABLISHED &&
1100 tp->t_rxt_minimum_timeout > 0) {
1101 /*
1102 * Avoid dropping a connection if minimum
1103 * timeout is set and that time did not
1104 * pass. We will retry sending
1105 * retransmissions at the maximum interval
1106 */
1107 if (TSTMP_LT(tcp_now, (tp->t_rxtstart +
1108 tp->t_rxt_minimum_timeout))) {
1109 tp->t_rxtshift = TCP_MAXRXTSHIFT - 1;
1110 goto retransmit_packet;
1111 }
1112 }
1113 if ((tp->t_flagsext & TF_RXTFINDROP) != 0) {
1114 tcpstat.tcps_rxtfindrop++;
1115 } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) {
1116 tcpstat.tcps_drop_after_sleep++;
1117 } else {
1118 tcpstat.tcps_timeoutdrop++;
1119 }
1120 if (tp->t_rxtshift >= TCP_MAXRXTSHIFT) {
1121 if (TCP_ECN_ENABLED(tp)) {
1122 INP_INC_IFNET_STAT(tp->t_inpcb,
1123 ecn_on.rxmit_drop);
1124 } else {
1125 INP_INC_IFNET_STAT(tp->t_inpcb,
1126 ecn_off.rxmit_drop);
1127 }
1128 }
1129 tp->t_rxtshift = TCP_MAXRXTSHIFT;
1130 soevent(so,
1131 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1132
1133 if (TCP_ECN_ENABLED(tp) &&
1134 tp->t_state == TCPS_ESTABLISHED) {
1135 tcp_heuristic_ecn_droprxmt(tp);
1136 }
1137
1138 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1139 "retransmission timeout drop");
1140 tp = tcp_drop(tp, tp->t_softerror ?
1141 tp->t_softerror : ETIMEDOUT);
1142
1143 break;
1144 }
1145 retransmit_packet:
1146 tcpstat.tcps_rexmttimeo++;
1147 tp->t_accsleep_ms = accsleep_ms;
1148
1149 if (tp->t_rxtshift == 1 &&
1150 tp->t_state == TCPS_ESTABLISHED) {
1151 /* Set the time at which retransmission started. */
1152 tp->t_rxtstart = tcp_now;
1153
1154 /*
1155 * if this is the first retransmit timeout, save
1156 * the state so that we can recover if the timeout
1157 * is spurious.
1158 */
1159 tcp_rexmt_save_state(tp);
1160 tcp_ccdbg_trace(tp, NULL, TCP_CC_FIRST_REXMT);
1161 }
1162 #if MPTCP
1163 if ((tp->t_rxtshift >= mptcp_fail_thresh) &&
1164 (tp->t_state == TCPS_ESTABLISHED) &&
1165 (tp->t_mpflags & TMPF_MPTCP_TRUE)) {
1166 mptcp_act_on_txfail(so);
1167 }
1168
1169 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1170 (so->so_flags & SOF_MP_SUBFLOW)) {
1171 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1172
1173 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER ||
1174 mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
1175 mptcp_check_subflows_and_add(mpte);
1176 }
1177 }
1178 #endif /* MPTCP */
1179
1180 if (tp->t_adaptive_wtimo > 0 &&
1181 tp->t_rxtshift > tp->t_adaptive_wtimo &&
1182 TCPS_HAVEESTABLISHED(tp->t_state)) {
1183 /* Send an event to the application */
1184 soevent(so,
1185 (SO_FILT_HINT_LOCKED |
1186 SO_FILT_HINT_ADAPTIVE_WTIMO));
1187 }
1188
1189 /*
1190 * If this is a retransmit timeout after PTO, the PTO
1191 * was not effective
1192 */
1193 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1194 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1195 tcpstat.tcps_rto_after_pto++;
1196 }
1197
1198 if (tp->t_flagsext & TF_DELAY_RECOVERY) {
1199 /*
1200 * Retransmit timer fired before entering recovery
1201 * on a connection with packet re-ordering. This
1202 * suggests that the reordering metrics computed
1203 * are not accurate.
1204 */
1205 tp->t_reorderwin = 0;
1206 tp->t_timer[TCPT_DELAYFR] = 0;
1207 tp->t_flagsext &= ~(TF_DELAY_RECOVERY);
1208 }
1209
1210 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1211 tp->t_state == TCPS_SYN_RECEIVED) {
1212 tcp_disable_tfo(tp);
1213 }
1214
1215 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1216 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1217 (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
1218 !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
1219 ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||
1220 tp->t_rxtshift > 4)) {
1221 /*
1222 * For regular retransmissions, a first one is being
1223 * done for tail-loss probe.
1224 * Thus, if rxtshift > 1, this means we have sent the segment
1225 * a total of 3 times.
1226 *
1227 * If we are in SYN-SENT state, then there is no tail-loss
1228 * probe thus we have to let rxtshift go up to 3.
1229 */
1230 tcp_heuristic_tfo_middlebox(tp);
1231
1232 so->so_error = ENODATA;
1233 soevent(so,
1234 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1235 sorwakeup(so);
1236 sowwakeup(so);
1237
1238 tp->t_tfo_stats |= TFO_S_SEND_BLACKHOLE;
1239 tcpstat.tcps_tfo_sndblackhole++;
1240 }
1241
1242 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1243 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1244 (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&
1245 tp->t_rxtshift > 3) {
1246 if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
1247 tcp_heuristic_tfo_middlebox(tp);
1248
1249 so->so_error = ENODATA;
1250 soevent(so,
1251 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1252 sorwakeup(so);
1253 sowwakeup(so);
1254 }
1255 }
1256
1257 if (tp->t_state == TCPS_SYN_SENT) {
1258 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_SYNRMXT) != 0 &&
1259 if_link_heuristics_enabled(outifp)) {
1260 IF_TCP_STATINC(outifp, linkheur_synrxmt);
1261 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1262 } else {
1263 rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift];
1264 }
1265 tp->t_stat.synrxtshift = tp->t_rxtshift;
1266 tp->t_stat.rxmitsyns++;
1267
1268 /* When retransmitting, disable TFO */
1269 if (TFO_ENABLED(tp) &&
1270 !(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE)) {
1271 tcp_disable_tfo(tp);
1272 tp->t_tfo_flags |= TFO_F_SYN_LOSS;
1273 }
1274 } else {
1275 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1276 }
1277 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX,
1278 TCP_ADD_REXMTSLOP(tp));
1279
1280 tcp_set_link_heur_rtomin(tp, outifp);
1281 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1282
1283 TCP_LOG_RTT_INFO(tp);
1284
1285 if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb)) {
1286 goto fc_output;
1287 }
1288
1289 tcp_free_sackholes(tp);
1290 if (TCP_RACK_ENABLED(tp)) {
1291 tcp_segs_clear_sacked(tp);
1292 tcp_rack_loss_on_rto(tp, true);
1293 }
1294 /*
1295 * Check for potential Path MTU Discovery Black Hole
1296 */
1297 if (tcp_pmtud_black_hole_detect &&
1298 !(tp->t_flagsext & TF_NOBLACKHOLE_DETECTION) &&
1299 (tp->t_state == TCPS_ESTABLISHED)) {
1300 if ((tp->t_flags & TF_PMTUD) &&
1301 tp->t_pmtud_lastseg_size > tcp_pmtud_black_holed_next_mss(tp) &&
1302 tp->t_rxtshift == 2) {
1303 /*
1304 * Enter Path MTU Black-hole Detection mechanism:
1305 * - Disable Path MTU Discovery (IP "DF" bit).
1306 * - Reduce MTU to lower value than what we
1307 * negotiated with the peer.
1308 */
1309 /* Disable Path MTU Discovery for now */
1310 tp->t_flags &= ~TF_PMTUD;
1311 /* Record that we may have found a black hole */
1312 tp->t_flags |= TF_BLACKHOLE;
1313 optlen = tp->t_maxopd - tp->t_maxseg;
1314 /* Keep track of previous MSS */
1315 tp->t_pmtud_saved_maxopd = tp->t_maxopd;
1316 tp->t_pmtud_start_ts = tcp_now;
1317 if (tp->t_pmtud_start_ts == 0) {
1318 tp->t_pmtud_start_ts++;
1319 }
1320 /* Reduce the MSS to intermediary value */
1321 tp->t_maxopd = tcp_pmtud_black_holed_next_mss(tp);
1322 tp->t_maxseg = tp->t_maxopd - optlen;
1323
1324 /*
1325 * Reset the slow-start flight size
1326 * as it may depend on the new MSS
1327 */
1328 if (CC_ALGO(tp)->cwnd_init != NULL) {
1329 CC_ALGO(tp)->cwnd_init(tp);
1330 }
1331 tp->snd_cwnd = tp->t_maxseg;
1332
1333 if (TCP_USE_RLEDBAT(tp, so) &&
1334 tcp_cc_rledbat.rwnd_init != NULL) {
1335 tcp_cc_rledbat.rwnd_init(tp);
1336 }
1337 }
1338 /*
1339 * If further retransmissions are still
1340 * unsuccessful with a lowered MTU, maybe this
1341 * isn't a Black Hole and we restore the previous
1342 * MSS and blackhole detection flags.
1343 */
1344 else {
1345 if ((tp->t_flags & TF_BLACKHOLE) &&
1346 (tp->t_rxtshift > 4)) {
1347 tcp_pmtud_revert_segment_size(tp);
1348 tp->snd_cwnd = tp->t_maxseg;
1349 }
1350 }
1351 }
1352
1353 /*
1354 * Disable rfc1323 and rfc1644 if we haven't got any
1355 * response to our SYN (after we reach the threshold)
1356 * to work-around some broken terminal servers (most of
1357 * which have hopefully been retired) that have bad VJ
1358 * header compression code which trashes TCP segments
1359 * containing unknown-to-them TCP options.
1360 * Do this only on non-local connections.
1361 */
1362 if (tp->t_state == TCPS_SYN_SENT &&
1363 tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) {
1364 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP);
1365 }
1366
1367 /*
1368 * If losing, let the lower level know and try for
1369 * a better route. Also, if we backed off this far,
1370 * our srtt estimate is probably bogus. Clobber it
1371 * so we'll take the next rtt measurement as our srtt;
1372 * move the current srtt into rttvar to keep the current
1373 * retransmit times until then.
1374 */
1375 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
1376 if (!(tp->t_inpcb->inp_vflag & INP_IPV4)) {
1377 in6_losing(tp->t_inpcb);
1378 } else {
1379 in_losing(tp->t_inpcb);
1380 }
1381 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
1382 tp->t_srtt = 0;
1383 }
1384 tp->snd_nxt = tp->snd_una;
1385 /*
1386 * Note: We overload snd_recover to function also as the
1387 * snd_last variable described in RFC 2582
1388 */
1389 tp->snd_recover = tp->snd_max;
1390 /*
1391 * Force a segment to be sent.
1392 */
1393 tp->t_flags |= TF_ACKNOW;
1394
1395 /*
1396 * If timing a segment in this window, stop the timer
1397 * except when we are in connecting states on cellular
1398 * interfaces
1399 */
1400 if (tp->t_state >= TCPS_ESTABLISHED || (outifp != NULL &&
1401 IFNET_IS_CELLULAR(outifp) == false)) {
1402 tp->t_rtttime = 0;
1403 }
1404
1405 if (!IN_FASTRECOVERY(tp) && tp->t_rxtshift == 1) {
1406 tcpstat.tcps_tailloss_rto++;
1407 }
1408
1409 /*
1410 * RFC 5681 says: when a TCP sender detects segment loss
1411 * using retransmit timer and the given segment has already
1412 * been retransmitted by way of the retransmission timer at
1413 * least once, the value of ssthresh is held constant
1414 */
1415 if (tp->t_rxtshift == 1 &&
1416 CC_ALGO(tp)->after_timeout != NULL) {
1417 CC_ALGO(tp)->after_timeout(tp);
1418 /*
1419 * CWR notifications are to be sent on new data
1420 * right after Fast Retransmits and ECE
1421 * notification receipts.
1422 */
1423 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1424 tp->ecn_flags |= TE_SENDCWR;
1425 }
1426 }
1427
1428 EXIT_FASTRECOVERY(tp);
1429
1430 /* Exit cwnd non validated phase */
1431 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
1432
1433
1434 fc_output:
1435 tcp_ccdbg_trace(tp, NULL, TCP_CC_REXMT_TIMEOUT);
1436
1437 (void) tcp_output(tp);
1438 break;
1439
1440 /*
1441 * Persistance timer into zero window.
1442 * Force a byte to be output, if possible.
1443 */
1444 case TCPT_PERSIST:
1445 tcpstat.tcps_persisttimeo++;
1446 /*
1447 * Hack: if the peer is dead/unreachable, we do not
1448 * time out if the window is closed. After a full
1449 * backoff, drop the connection if the idle time
1450 * (no responses to probes) reaches the maximum
1451 * backoff that we would use if retransmitting.
1452 *
1453 * Drop the connection if we reached the maximum allowed time for
1454 * Zero Window Probes without a non-zero update from the peer.
1455 * See rdar://5805356
1456 */
1457 if ((tp->t_rxtshift == TCP_MAXRXTSHIFT &&
1458 (idle_time >= tcp_maxpersistidle ||
1459 idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) ||
1460 ((tp->t_persist_stop != 0) &&
1461 TSTMP_LEQ(tp->t_persist_stop, tcp_now))) {
1462 TCP_LOG_DROP_PCB(NULL, NULL, tp, false, "persist timeout drop");
1463 tcpstat.tcps_persistdrop++;
1464 soevent(so,
1465 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1466 tp = tcp_drop(tp, ETIMEDOUT);
1467 break;
1468 }
1469 tcp_setpersist(tp);
1470 tp->t_flagsext |= TF_FORCE;
1471 (void) tcp_output(tp);
1472 tp->t_flagsext &= ~TF_FORCE;
1473 break;
1474
1475 /*
1476 * Keep-alive timer went off; send something
1477 * or drop connection if idle for too long.
1478 */
1479 case TCPT_KEEP:
1480 #if FLOW_DIVERT
1481 if (tp->t_inpcb->inp_socket->so_flags & SOF_FLOW_DIVERT) {
1482 break;
1483 }
1484 #endif /* FLOW_DIVERT */
1485
1486 tcpstat.tcps_keeptimeo++;
1487 #if MPTCP
1488 /*
1489 * Regular TCP connections do not send keepalives after closing
1490 * MPTCP must not also, after sending Data FINs.
1491 */
1492 struct mptcb *mp_tp = tptomptp(tp);
1493 if ((tp->t_mpflags & TMPF_MPTCP_TRUE) &&
1494 (tp->t_state > TCPS_ESTABLISHED)) {
1495 goto dropit;
1496 } else if (mp_tp != NULL) {
1497 if ((mptcp_ok_to_keepalive(mp_tp) == 0)) {
1498 goto dropit;
1499 }
1500 }
1501 #endif /* MPTCP */
1502 if (tp->t_state < TCPS_ESTABLISHED) {
1503 goto dropit;
1504 }
1505 if ((always_keepalive ||
1506 (tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) ||
1507 (tp->t_flagsext & TF_DETECT_READSTALL) ||
1508 (tp->t_tfo_probe_state == TFO_PROBE_PROBING)) &&
1509 (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
1510 if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp)) {
1511 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1512 "keep alive timeout drop");
1513 goto dropit;
1514 }
1515
1516 if (tcp_send_keep_alive(tp)) {
1517 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1518 tp->t_rtimo_probes++;
1519 }
1520
1521 TCP_LOG_KEEP_ALIVE(tp, idle_time);
1522 }
1523
1524 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1525 TCP_CONN_KEEPINTVL(tp));
1526 } else {
1527 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1528 TCP_CONN_KEEPIDLE(tp));
1529 }
1530 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1531 bool reenable_probe = false;
1532 /*
1533 * The keep alive packets sent to detect a read
1534 * stall did not get a response from the
1535 * peer. Generate more keep-alives to confirm this.
1536 * If the number of probes sent reaches the limit,
1537 * generate an event.
1538 */
1539 if (tp->t_adaptive_rtimo > 0) {
1540 if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) {
1541 /* Generate an event */
1542 soevent(so,
1543 (SO_FILT_HINT_LOCKED |
1544 SO_FILT_HINT_ADAPTIVE_RTIMO));
1545 tcp_keepalive_reset(tp);
1546 } else {
1547 reenable_probe = true;
1548 }
1549 } else if (outifp != NULL &&
1550 (outifp->if_eflags & IFEF_PROBE_CONNECTIVITY) &&
1551 tp->t_rtimo_probes <= TCP_CONNECTIVITY_PROBES_MAX) {
1552 reenable_probe = true;
1553 } else {
1554 tp->t_flagsext &= ~TF_DETECT_READSTALL;
1555 }
1556 if (reenable_probe) {
1557 int ind = min(tp->t_rtimo_probes,
1558 TCP_MAXRXTSHIFT);
1559 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(
1560 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp));
1561 }
1562 }
1563 if (tp->t_tfo_probe_state == TFO_PROBE_PROBING) {
1564 int ind;
1565
1566 tp->t_tfo_probes++;
1567 ind = min(tp->t_tfo_probes, TCP_MAXRXTSHIFT);
1568
1569 /*
1570 * We take the minimum among the time set by true
1571 * keepalive (see above) and the backoff'd RTO. That
1572 * way we backoff in case of packet-loss but will never
1573 * timeout slower than regular keepalive due to the
1574 * backing off.
1575 */
1576 tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START(
1577 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)),
1578 tp->t_timer[TCPT_KEEP]);
1579 } else if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1580 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1581 tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) {
1582 /* Still no data! Let's assume a TFO-error and err out... */
1583 tcp_heuristic_tfo_middlebox(tp);
1584
1585 so->so_error = ENODATA;
1586 soevent(so,
1587 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1588 sorwakeup(so);
1589 tp->t_tfo_stats |= TFO_S_RECV_BLACKHOLE;
1590 tcpstat.tcps_tfo_blackhole++;
1591 }
1592 break;
1593 case TCPT_DELACK:
1594 if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) {
1595 tp->t_flags &= ~TF_DELACK;
1596 tp->t_timer[TCPT_DELACK] = 0;
1597 tp->t_flags |= TF_ACKNOW;
1598
1599 /*
1600 * If delayed ack timer fired while stretching
1601 * acks, count the number of times the streaming
1602 * detection was not correct. If this exceeds a
1603 * threshold, disable strech ack on this
1604 * connection
1605 *
1606 * Also, go back to acking every other packet.
1607 */
1608 if ((tp->t_flags & TF_STRETCHACK)) {
1609 if (tp->t_unacksegs > 1 &&
1610 tp->t_unacksegs < maxseg_unacked) {
1611 tp->t_stretchack_delayed++;
1612 }
1613
1614 if (tp->t_stretchack_delayed >
1615 TCP_STRETCHACK_DELAY_THRESHOLD) {
1616 tp->t_flagsext |= TF_DISABLE_STRETCHACK;
1617 /*
1618 * Note the time at which stretch
1619 * ack was disabled automatically
1620 */
1621 tp->rcv_nostrack_ts = tcp_now;
1622 tcpstat.tcps_nostretchack++;
1623 tp->t_stretchack_delayed = 0;
1624 tp->rcv_nostrack_pkts = 0;
1625 }
1626 tcp_reset_stretch_ack(tp);
1627 }
1628 tp->t_forced_acks = TCP_FORCED_ACKS_COUNT;
1629
1630 /*
1631 * If we are measuring inter packet arrival jitter
1632 * for throttling a connection, this delayed ack
1633 * might be the reason for accumulating some
1634 * jitter. So let's restart the measurement.
1635 */
1636 CLEAR_IAJ_STATE(tp);
1637
1638 tcpstat.tcps_delack++;
1639 tp->t_stat.delayed_acks_sent++;
1640 (void) tcp_output(tp);
1641 }
1642 break;
1643
1644 #if MPTCP
1645 case TCPT_JACK_RXMT:
1646 if ((tp->t_state == TCPS_ESTABLISHED) &&
1647 (tp->t_mpflags & TMPF_PREESTABLISHED) &&
1648 (tp->t_mpflags & TMPF_JOINED_FLOW)) {
1649 if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) {
1650 tcpstat.tcps_timeoutdrop++;
1651 soevent(so,
1652 (SO_FILT_HINT_LOCKED |
1653 SO_FILT_HINT_TIMEOUT));
1654 tp = tcp_drop(tp, tp->t_softerror ?
1655 tp->t_softerror : ETIMEDOUT);
1656 break;
1657 }
1658 tcpstat.tcps_join_rxmts++;
1659 tp->t_mpflags |= TMPF_SND_JACK;
1660 tp->t_flags |= TF_ACKNOW;
1661
1662 /*
1663 * No backoff is implemented for simplicity for this
1664 * corner case.
1665 */
1666 (void) tcp_output(tp);
1667 }
1668 break;
1669 case TCPT_CELLICON:
1670 {
1671 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1672
1673 tp->t_timer[TCPT_CELLICON] = 0;
1674
1675 if (mpte->mpte_cellicon_increments == 0) {
1676 /* Cell-icon not set by this connection */
1677 break;
1678 }
1679
1680 if (TSTMP_LT(mpte->mpte_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, tcp_now)) {
1681 mptcp_unset_cellicon(mpte, NULL, 1);
1682 }
1683
1684 if (mpte->mpte_cellicon_increments) {
1685 tp->t_timer[TCPT_CELLICON] = OFFSET_FROM_START(tp, MPTCP_CELLICON_TOGGLE_RATE);
1686 }
1687
1688 break;
1689 }
1690 #endif /* MPTCP */
1691
1692 case TCPT_PTO:
1693 {
1694 int32_t ret = 0;
1695
1696 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1697 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1698 }
1699 /*
1700 * Check if the connection is in the right state to
1701 * send a probe
1702 */
1703 if ((tp->t_state != TCPS_ESTABLISHED ||
1704 tp->t_rxtshift > 0 ||
1705 tp->snd_max == tp->snd_una ||
1706 !SACK_ENABLED(tp) || IN_FASTRECOVERY(tp)) &&
1707 !(tp->t_flagsext & TF_IF_PROBING)) {
1708 break;
1709 }
1710
1711 /*
1712 * When the interface state is changed explicitly reset the retransmission
1713 * timer state for both SYN and data packets because we do not want to
1714 * wait unnecessarily or timeout too quickly if the link characteristics
1715 * have changed drastically
1716 */
1717 if (tp->t_flagsext & TF_IF_PROBING) {
1718 tp->t_rxtshift = 0;
1719 if (tp->t_state == TCPS_SYN_SENT) {
1720 tp->t_stat.synrxtshift = tp->t_rxtshift;
1721 }
1722 /*
1723 * Reset to the the default RTO
1724 */
1725 tp->t_srtt = TCPTV_SRTTBASE;
1726 tp->t_rttvar =
1727 ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
1728 tp->t_rttmin = TCPTV_REXMTMIN;
1729 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
1730 tp->t_rttmin, TCPTV_REXMTMAX, TCP_ADD_REXMTSLOP(tp));
1731 TCP_LOG_RTT_INFO(tp);
1732 }
1733
1734 if (tp->t_state == TCPS_SYN_SENT) {
1735 /*
1736 * The PTO for SYN_SENT reinitializes TCP as if it was a fresh
1737 * connection attempt
1738 */
1739 tp->snd_nxt = tp->snd_una;
1740 /*
1741 * Note: We overload snd_recover to function also as the
1742 * snd_last variable described in RFC 2582
1743 */
1744 tp->snd_recover = tp->snd_max;
1745 /*
1746 * Force a segment to be sent.
1747 */
1748 tp->t_flags |= TF_ACKNOW;
1749
1750 /* If timing a segment in this window, stop the timer */
1751 tp->t_rtttime = 0;
1752
1753 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1754 } else {
1755 int32_t snd_len;
1756
1757 /*
1758 * If there is no new data to send or if the
1759 * connection is limited by receive window then
1760 * retransmit the last segment, otherwise send
1761 * new data.
1762 */
1763 snd_len = min(so->so_snd.sb_cc, tp->snd_wnd)
1764 - (tp->snd_max - tp->snd_una);
1765 if (snd_len > 0) {
1766 tp->snd_nxt = tp->snd_max;
1767 tp->t_flagsext &= ~TF_TLP_IS_RETRANS;
1768 } else {
1769 snd_len = min((tp->snd_max - tp->snd_una),
1770 tp->t_maxseg);
1771 tp->snd_nxt = tp->snd_max - snd_len;
1772 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1773 }
1774 }
1775
1776 tcpstat.tcps_pto++;
1777 if (tp->t_flagsext & TF_IF_PROBING) {
1778 tcpstat.tcps_probe_if++;
1779 }
1780
1781 /* If timing a segment in this window, stop the timer */
1782 tp->t_rtttime = 0;
1783 /* Note that tail loss probe is being sent. Exclude IF probe */
1784 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1785 tp->t_flagsext |= TF_SENT_TLPROBE;
1786 tp->t_tlpstart = tcp_now;
1787 }
1788
1789 tp->snd_cwnd += tp->t_maxseg;
1790 /*
1791 * When tail-loss-probe fires, we reset the RTO timer, because
1792 * a probe just got sent, so we are good to push out the timer.
1793 *
1794 * Set to 0 to ensure that tcp_output() will reschedule it
1795 */
1796 tp->t_timer[TCPT_REXMT] = 0;
1797 ret = tcp_output(tp);
1798
1799 #if (DEBUG || DEVELOPMENT)
1800 if ((tp->t_flagsext & TF_IF_PROBING) &&
1801 ((IFNET_IS_COMPANION_LINK(tp->t_inpcb->inp_last_outifp)) ||
1802 tp->t_state == TCPS_SYN_SENT)) {
1803 if (ret == 0 && tcp_probe_if_fix_port > 0 &&
1804 tcp_probe_if_fix_port <= IPPORT_HILASTAUTO) {
1805 tp->t_timer[TCPT_REXMT] = 0;
1806 tcp_set_lotimer_index(tp);
1807 }
1808
1809 os_log(OS_LOG_DEFAULT,
1810 "%s: sent %s probe for %u > %u on interface %s"
1811 " (%u) %s(%d)",
1812 __func__,
1813 tp->t_state == TCPS_SYN_SENT ? "SYN" : "data",
1814 ntohs(tp->t_inpcb->inp_lport),
1815 ntohs(tp->t_inpcb->inp_fport),
1816 if_name(tp->t_inpcb->inp_last_outifp),
1817 tp->t_inpcb->inp_last_outifp->if_index,
1818 ret == 0 ? "succeeded" :"failed", ret);
1819 }
1820 #endif /* DEBUG || DEVELOPMENT */
1821
1822 /*
1823 * When there is data (or a SYN) to send, the above call to
1824 * tcp_output() should have armed either the REXMT or the
1825 * PERSIST timer. If it didn't, something is wrong and this
1826 * connection would idle around forever. Let's make sure that
1827 * at least the REXMT timer is set.
1828 */
1829 if (tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0 &&
1830 (tp->t_inpcb->inp_socket->so_snd.sb_cc != 0 || tp->t_state == TCPS_SYN_SENT ||
1831 tp->t_state == TCPS_SYN_RECEIVED)) {
1832 tcp_set_link_heur_rtomin(tp, outifp);
1833 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1834
1835 os_log(OS_LOG_DEFAULT,
1836 "%s: tcp_output() returned %u with retransmission timer disabled "
1837 "for %u > %u in state %d, reset timer to %d",
1838 __func__, ret,
1839 ntohs(tp->t_inpcb->inp_lport),
1840 ntohs(tp->t_inpcb->inp_fport),
1841 tp->t_state,
1842 tp->t_timer[TCPT_REXMT]);
1843
1844 tcp_check_timer_state(tp);
1845 }
1846 tp->snd_cwnd -= tp->t_maxseg;
1847
1848 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1849 tp->t_tlphighrxt = tp->snd_nxt;
1850 tp->t_tlphightrxt_persist = tp->snd_nxt;
1851 }
1852 break;
1853 }
1854 case TCPT_DELAYFR:
1855 tp->t_flagsext &= ~TF_DELAY_RECOVERY;
1856
1857 /*
1858 * Don't do anything if one of the following is true:
1859 * - the connection is already in recovery
1860 * - sequence until snd_recover has been acknowledged.
1861 * - retransmit timeout has fired
1862 */
1863 if (IN_FASTRECOVERY(tp) ||
1864 SEQ_GEQ(tp->snd_una, tp->snd_recover) ||
1865 tp->t_rxtshift > 0) {
1866 break;
1867 }
1868
1869 VERIFY(SACK_ENABLED(tp));
1870 tcp_rexmt_save_state(tp);
1871 if (CC_ALGO(tp)->pre_fr != NULL) {
1872 CC_ALGO(tp)->pre_fr(tp);
1873 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1874 tp->ecn_flags |= TE_SENDCWR;
1875 }
1876 }
1877 ENTER_FASTRECOVERY(tp);
1878
1879 tp->t_timer[TCPT_REXMT] = 0;
1880 tcpstat.tcps_sack_recovery_episode++;
1881 tp->t_sack_recovery_episode++;
1882 tp->snd_cwnd = tp->t_maxseg;
1883 tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY);
1884 (void) tcp_output(tp);
1885 break;
1886
1887 dropit:
1888 tcpstat.tcps_keepdrops++;
1889 soevent(so,
1890 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1891 tp = tcp_drop(tp, ETIMEDOUT);
1892 break;
1893 case TCPT_REORDER:
1894 if (TCP_RACK_ENABLED(tp)) {
1895 tcp_rack_reordering_timeout(tp, 0);
1896 }
1897 break;
1898 }
1899 return tp;
1900 }
1901
1902 /* Remove a timer entry from timer list */
1903 void
tcp_remove_timer(struct tcpcb * tp)1904 tcp_remove_timer(struct tcpcb *tp)
1905 {
1906 struct tcptimerlist *listp = &tcp_timer_list;
1907
1908 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
1909 if (!(TIMER_IS_ON_LIST(tp))) {
1910 return;
1911 }
1912 lck_mtx_lock(&listp->mtx);
1913
1914 if (listp->next_te != NULL && listp->next_te == &tp->tentry) {
1915 listp->next_te = LIST_NEXT(&tp->tentry, le);
1916 }
1917
1918 LIST_REMOVE(&tp->tentry, le);
1919 /*
1920 * The use count has been incremented when the PCB
1921 * was placed on the timer list, and needs to be decremented.
1922 * As a safety precaution, we are checking against underflow.
1923 */
1924 if (__improbable(tp->t_inpcb->inp_socket->so_usecount == 0)) {
1925 TCP_LOG(tp, "%s: inpcb socket so_usecount underflow "
1926 " when removing timer entry\n", __func__);
1927 } else {
1928 tp->t_inpcb->inp_socket->so_usecount--;
1929 }
1930
1931 tp->t_flags &= ~(TF_TIMER_ONLIST);
1932
1933 listp->entries--;
1934
1935 tp->tentry.le.le_next = NULL;
1936 tp->tentry.le.le_prev = NULL;
1937
1938 lck_mtx_unlock(&listp->mtx);
1939 }
1940
1941 /*
1942 * Function to check if the timerlist needs to be rescheduled to run
1943 * the timer entry correctly. Basically, this is to check if we can avoid
1944 * taking the list lock.
1945 */
1946
1947 static boolean_t
need_to_resched_timerlist(u_int32_t runtime,u_int16_t mode)1948 need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode)
1949 {
1950 struct tcptimerlist *listp = &tcp_timer_list;
1951 int32_t diff;
1952
1953 /*
1954 * If the list is being processed then the state of the list is
1955 * in flux. In this case always acquire the lock and set the state
1956 * correctly.
1957 */
1958 if (listp->running) {
1959 return TRUE;
1960 }
1961
1962 if (!listp->scheduled) {
1963 return TRUE;
1964 }
1965
1966 diff = timer_diff(listp->runtime, 0, runtime, 0);
1967 if (diff <= 0) {
1968 /* The list is going to run before this timer */
1969 return FALSE;
1970 } else {
1971 if (mode & TCP_TIMERLIST_10MS_MODE) {
1972 if (diff <= TCP_TIMER_10MS_QUANTUM) {
1973 return FALSE;
1974 }
1975 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
1976 if (diff <= TCP_TIMER_100MS_QUANTUM) {
1977 return FALSE;
1978 }
1979 } else {
1980 if (diff <= TCP_TIMER_500MS_QUANTUM) {
1981 return FALSE;
1982 }
1983 }
1984 }
1985 return TRUE;
1986 }
1987
1988 void
tcp_sched_timerlist(uint32_t offset)1989 tcp_sched_timerlist(uint32_t offset)
1990 {
1991 uint64_t deadline = 0;
1992 struct tcptimerlist *listp = &tcp_timer_list;
1993
1994 LCK_MTX_ASSERT(&listp->mtx, LCK_MTX_ASSERT_OWNED);
1995
1996 offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);
1997 listp->runtime = tcp_now + offset;
1998 listp->schedtime = tcp_now;
1999 if (listp->runtime == 0) {
2000 listp->runtime++;
2001 offset++;
2002 }
2003
2004 clock_interval_to_deadline(offset, USEC_PER_SEC, &deadline);
2005
2006 thread_call_enter_delayed(listp->call, deadline);
2007 listp->scheduled = TRUE;
2008 }
2009
2010 /*
2011 * Function to run the timers for a connection.
2012 *
2013 * Returns the offset of next timer to be run for this connection which
2014 * can be used to reschedule the timerlist.
2015 *
2016 * te_mode is an out parameter that indicates the modes of active
2017 * timers for this connection.
2018 */
2019 u_int32_t
tcp_run_conn_timer(struct tcpcb * tp,u_int16_t * te_mode,u_int16_t probe_if_index)2020 tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode,
2021 u_int16_t probe_if_index)
2022 {
2023 struct socket *so;
2024 u_int16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE;
2025 u_int32_t timer_val, offset = 0, lo_timer = 0;
2026 int32_t diff;
2027 boolean_t needtorun[TCPT_NTIMERS];
2028 int count = 0;
2029
2030 VERIFY(tp != NULL);
2031 bzero(needtorun, sizeof(needtorun));
2032 *te_mode = 0;
2033
2034 socket_lock(tp->t_inpcb->inp_socket, 1);
2035
2036 so = tp->t_inpcb->inp_socket;
2037 /* Release the want count on inp */
2038 if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1)
2039 == WNT_STOPUSING) {
2040 if (TIMER_IS_ON_LIST(tp)) {
2041 tcp_remove_timer(tp);
2042 }
2043
2044 /* Looks like the TCP connection got closed while we
2045 * were waiting for the lock.. Done
2046 */
2047 goto done;
2048 }
2049
2050 /*
2051 * If this connection is over an interface that needs to
2052 * be probed, send probe packets to reinitiate communication.
2053 */
2054 if (TCP_IF_STATE_CHANGED(tp, probe_if_index)) {
2055 tp->t_flagsext |= TF_IF_PROBING;
2056 tcp_timers(tp, TCPT_PTO);
2057 tp->t_timer[TCPT_PTO] = 0;
2058 tp->t_flagsext &= ~TF_IF_PROBING;
2059 }
2060
2061 /*
2062 * Since the timer thread needs to wait for tcp lock, it may race
2063 * with another thread that can cancel or reschedule the timer
2064 * that is about to run. Check if we need to run anything.
2065 */
2066 if ((index = tp->tentry.index) == TCPT_NONE) {
2067 goto done;
2068 }
2069
2070 timer_val = tp->t_timer[index];
2071
2072 diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
2073 if (diff > 0) {
2074 if (tp->tentry.index != TCPT_NONE) {
2075 offset = diff;
2076 *(te_mode) = tp->tentry.mode;
2077 }
2078 goto done;
2079 }
2080
2081 tp->t_timer[index] = 0;
2082 if (timer_val > 0) {
2083 tp = tcp_timers(tp, index);
2084 if (tp == NULL) {
2085 goto done;
2086 }
2087 }
2088
2089 /*
2090 * Check if there are any other timers that need to be run.
2091 * While doing it, adjust the timer values wrt tcp_now.
2092 */
2093 tp->tentry.mode = 0;
2094 for (i = 0; i < TCPT_NTIMERS; ++i) {
2095 if (tp->t_timer[i] != 0) {
2096 diff = timer_diff(tp->tentry.timer_start,
2097 tp->t_timer[i], tcp_now, 0);
2098 if (diff <= 0) {
2099 needtorun[i] = TRUE;
2100 count++;
2101 } else {
2102 tp->t_timer[i] = diff;
2103 needtorun[i] = FALSE;
2104 if (lo_timer == 0 || diff < lo_timer) {
2105 lo_timer = diff;
2106 lo_index = i;
2107 }
2108 TCP_SET_TIMER_MODE(tp->tentry.mode, i);
2109 }
2110 }
2111 }
2112
2113 tp->tentry.timer_start = tcp_now;
2114 tp->tentry.index = lo_index;
2115 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2116
2117 if (tp->tentry.index != TCPT_NONE) {
2118 tp->tentry.runtime = tp->tentry.timer_start +
2119 tp->t_timer[tp->tentry.index];
2120 if (tp->tentry.runtime == 0) {
2121 tp->tentry.runtime++;
2122 }
2123 }
2124
2125 if (count > 0) {
2126 /* run any other timers outstanding at this time. */
2127 for (i = 0; i < TCPT_NTIMERS; ++i) {
2128 if (needtorun[i]) {
2129 tp->t_timer[i] = 0;
2130 tp = tcp_timers(tp, i);
2131 if (tp == NULL) {
2132 offset = 0;
2133 *(te_mode) = 0;
2134 goto done;
2135 }
2136 }
2137 }
2138 tcp_set_lotimer_index(tp);
2139 }
2140
2141 if (tp->tentry.index < TCPT_NONE) {
2142 offset = tp->t_timer[tp->tentry.index];
2143 *(te_mode) = tp->tentry.mode;
2144 }
2145
2146 done:
2147 if (tp != NULL && tp->tentry.index == TCPT_NONE) {
2148 tcp_remove_timer(tp);
2149 offset = 0;
2150 }
2151
2152 socket_unlock(so, 1);
2153 return offset;
2154 }
2155
2156 void
tcp_run_timerlist(void * arg1,void * arg2)2157 tcp_run_timerlist(void * arg1, void * arg2)
2158 {
2159 #pragma unused(arg1, arg2)
2160 struct tcptimerentry *te, *__single next_te;
2161 struct tcptimerlist *__single listp = &tcp_timer_list;
2162 struct tcpcb *__single tp;
2163 uint32_t next_timer = 0; /* offset of the next timer on the list */
2164 u_int16_t te_mode = 0; /* modes of all active timers in a tcpcb */
2165 u_int16_t list_mode = 0; /* cumulative of modes of all tcpcbs */
2166 uint32_t active_count = 0;
2167
2168 calculate_tcp_clock();
2169
2170 lck_mtx_lock(&listp->mtx);
2171
2172 int32_t drift = tcp_now - listp->runtime;
2173 if (drift <= 1) {
2174 tcpstat.tcps_timer_drift_le_1_ms++;
2175 } else if (drift <= 10) {
2176 tcpstat.tcps_timer_drift_le_10_ms++;
2177 } else if (drift <= 20) {
2178 tcpstat.tcps_timer_drift_le_20_ms++;
2179 } else if (drift <= 50) {
2180 tcpstat.tcps_timer_drift_le_50_ms++;
2181 } else if (drift <= 100) {
2182 tcpstat.tcps_timer_drift_le_100_ms++;
2183 } else if (drift <= 200) {
2184 tcpstat.tcps_timer_drift_le_200_ms++;
2185 } else if (drift <= 500) {
2186 tcpstat.tcps_timer_drift_le_500_ms++;
2187 } else if (drift <= 1000) {
2188 tcpstat.tcps_timer_drift_le_1000_ms++;
2189 } else {
2190 tcpstat.tcps_timer_drift_gt_1000_ms++;
2191 }
2192
2193 listp->running = TRUE;
2194
2195 LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
2196 uint32_t offset = 0;
2197 uint32_t runtime = te->runtime;
2198
2199 tp = TIMERENTRY_TO_TP(te);
2200
2201 /*
2202 * An interface probe may need to happen before the previously scheduled runtime
2203 */
2204 if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now) &&
2205 !TCP_IF_STATE_CHANGED(tp, listp->probe_if_index)) {
2206 offset = timer_diff(runtime, 0, tcp_now, 0);
2207 if (next_timer == 0 || offset < next_timer) {
2208 next_timer = offset;
2209 }
2210 list_mode |= te->mode;
2211 continue;
2212 }
2213
2214 /*
2215 * Acquire an inp wantcnt on the inpcb so that the socket
2216 * won't get detached even if tcp_close is called
2217 */
2218 if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0)
2219 == WNT_STOPUSING) {
2220 /*
2221 * Need to take socket lock because it protects
2222 * TIMER_IS_ON_LIST
2223 */
2224 lck_mtx_unlock(&listp->mtx);
2225 socket_lock(tp->t_inpcb->inp_socket, 1);
2226 tcp_remove_timer(tp);
2227 socket_unlock(tp->t_inpcb->inp_socket, 1);
2228 lck_mtx_lock(&listp->mtx);
2229 continue;
2230 }
2231 active_count++;
2232
2233 /*
2234 * Store the next timerentry pointer before releasing the
2235 * list lock. If that entry has to be removed when we
2236 * release the lock, this pointer will be updated to the
2237 * element after that.
2238 */
2239 listp->next_te = next_te;
2240
2241 VERIFY_NEXT_LINK(&tp->tentry, le);
2242 VERIFY_PREV_LINK(&tp->tentry, le);
2243
2244 lck_mtx_unlock(&listp->mtx);
2245
2246 offset = tcp_run_conn_timer(tp, &te_mode,
2247 listp->probe_if_index);
2248
2249 lck_mtx_lock(&listp->mtx);
2250
2251 next_te = listp->next_te;
2252 listp->next_te = NULL;
2253
2254 if (offset > 0 && te_mode != 0) {
2255 list_mode |= te_mode;
2256
2257 if (next_timer == 0 || offset < next_timer) {
2258 next_timer = offset;
2259 }
2260 }
2261 }
2262
2263 if (!LIST_EMPTY(&listp->lhead)) {
2264 uint32_t next_mode = 0;
2265 if ((list_mode & TCP_TIMERLIST_10MS_MODE) ||
2266 (listp->pref_mode & TCP_TIMERLIST_10MS_MODE)) {
2267 next_mode = TCP_TIMERLIST_10MS_MODE;
2268 } else if ((list_mode & TCP_TIMERLIST_100MS_MODE) ||
2269 (listp->pref_mode & TCP_TIMERLIST_100MS_MODE)) {
2270 next_mode = TCP_TIMERLIST_100MS_MODE;
2271 } else {
2272 next_mode = TCP_TIMERLIST_500MS_MODE;
2273 }
2274
2275 if (next_mode != TCP_TIMERLIST_500MS_MODE) {
2276 listp->idleruns = 0;
2277 } else {
2278 /*
2279 * the next required mode is slow mode, but if
2280 * the last one was a faster mode and we did not
2281 * have enough idle runs, repeat the last mode.
2282 *
2283 * We try to keep the timer list in fast mode for
2284 * some idle time in expectation of new data.
2285 */
2286 if (listp->mode != next_mode &&
2287 listp->idleruns < timer_fastmode_idlemax) {
2288 listp->idleruns++;
2289 next_mode = listp->mode;
2290 next_timer = TCP_TIMER_100MS_QUANTUM;
2291 } else {
2292 listp->idleruns = 0;
2293 }
2294 }
2295 listp->mode = next_mode;
2296 if (listp->pref_offset != 0) {
2297 next_timer = min(listp->pref_offset, next_timer);
2298 }
2299
2300 if (listp->mode == TCP_TIMERLIST_500MS_MODE) {
2301 next_timer = max(next_timer,
2302 TCP_TIMER_500MS_QUANTUM);
2303 }
2304
2305 tcp_sched_timerlist(next_timer);
2306 } else {
2307 /*
2308 * No need to reschedule this timer, but always run
2309 * periodically at a much higher granularity.
2310 */
2311 tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);
2312 }
2313
2314 listp->running = FALSE;
2315 listp->pref_mode = 0;
2316 listp->pref_offset = 0;
2317 listp->probe_if_index = 0;
2318
2319 lck_mtx_unlock(&listp->mtx);
2320 }
2321
2322 /*
2323 * Function to check if the timerlist needs to be rescheduled to run this
2324 * connection's timers correctly.
2325 */
2326 void
tcp_sched_timers(struct tcpcb * tp)2327 tcp_sched_timers(struct tcpcb *tp)
2328 {
2329 struct tcptimerentry *te = &tp->tentry;
2330 u_int16_t index = te->index;
2331 u_int16_t mode = te->mode;
2332 struct tcptimerlist *listp = &tcp_timer_list;
2333 int32_t offset = 0;
2334 boolean_t list_locked = FALSE;
2335
2336 if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) {
2337 /* Just return without adding the dead pcb to the list */
2338 if (TIMER_IS_ON_LIST(tp)) {
2339 tcp_remove_timer(tp);
2340 }
2341 return;
2342 }
2343
2344 if (index == TCPT_NONE) {
2345 /* Nothing to run */
2346 tcp_remove_timer(tp);
2347 return;
2348 }
2349
2350 /*
2351 * compute the offset at which the next timer for this connection
2352 * has to run.
2353 */
2354 offset = timer_diff(te->runtime, 0, tcp_now, 0);
2355 if (offset <= 0) {
2356 offset = 1;
2357 tcp_timer_advanced++;
2358 }
2359
2360 if (!TIMER_IS_ON_LIST(tp)) {
2361 if (!list_locked) {
2362 lck_mtx_lock(&listp->mtx);
2363 list_locked = TRUE;
2364 }
2365
2366 if (!TIMER_IS_ON_LIST(tp)) {
2367 /*
2368 * Adding the timer entry should constitute an incresed socket use count,
2369 * otherwise the socket use count may reach zero while being referenced
2370 * via the timer entry. If this happens, the timer service routine
2371 * will run into an UAF (use after free) when attempting
2372 * to get the related protocol control block.
2373 */
2374 tp->t_inpcb->inp_socket->so_usecount++;
2375
2376 LIST_INSERT_HEAD(&listp->lhead, te, le);
2377 tp->t_flags |= TF_TIMER_ONLIST;
2378
2379 listp->entries++;
2380 if (listp->entries > listp->maxentries) {
2381 listp->maxentries = listp->entries;
2382 }
2383
2384 /* if the list is not scheduled, just schedule it */
2385 if (!listp->scheduled) {
2386 goto schedule;
2387 }
2388 }
2389 }
2390
2391 /*
2392 * Timer entry is currently on the list, check if the list needs
2393 * to be rescheduled.
2394 */
2395 if (need_to_resched_timerlist(te->runtime, mode)) {
2396 tcp_resched_timerlist++;
2397
2398 if (!list_locked) {
2399 lck_mtx_lock(&listp->mtx);
2400 list_locked = TRUE;
2401 }
2402
2403 VERIFY_NEXT_LINK(te, le);
2404 VERIFY_PREV_LINK(te, le);
2405
2406 if (listp->running) {
2407 listp->pref_mode |= mode;
2408 if (listp->pref_offset == 0 ||
2409 offset < listp->pref_offset) {
2410 listp->pref_offset = offset;
2411 }
2412 } else {
2413 /*
2414 * The list could have got rescheduled while
2415 * this thread was waiting for the lock
2416 */
2417 if (listp->scheduled) {
2418 int32_t diff;
2419 diff = timer_diff(listp->runtime, 0,
2420 tcp_now, offset);
2421 if (diff <= 0) {
2422 goto done;
2423 } else {
2424 goto schedule;
2425 }
2426 } else {
2427 goto schedule;
2428 }
2429 }
2430 }
2431 goto done;
2432
2433 schedule:
2434 /*
2435 * Since a connection with timers is getting scheduled, the timer
2436 * list moves from idle to active state and that is why idlegen is
2437 * reset
2438 */
2439 if (mode & TCP_TIMERLIST_10MS_MODE) {
2440 listp->mode = TCP_TIMERLIST_10MS_MODE;
2441 listp->idleruns = 0;
2442 offset = min(offset, TCP_TIMER_10MS_QUANTUM);
2443 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
2444 if (listp->mode > TCP_TIMERLIST_100MS_MODE) {
2445 listp->mode = TCP_TIMERLIST_100MS_MODE;
2446 }
2447 listp->idleruns = 0;
2448 offset = min(offset, TCP_TIMER_100MS_QUANTUM);
2449 }
2450 tcp_sched_timerlist(offset);
2451
2452 done:
2453 if (list_locked) {
2454 lck_mtx_unlock(&listp->mtx);
2455 }
2456
2457 return;
2458 }
2459
2460 static inline void
tcp_set_lotimer_index(struct tcpcb * tp)2461 tcp_set_lotimer_index(struct tcpcb *tp)
2462 {
2463 uint16_t i, lo_index = TCPT_NONE, mode = 0;
2464 uint32_t lo_timer = 0;
2465 for (i = 0; i < TCPT_NTIMERS; ++i) {
2466 if (tp->t_timer[i] != 0) {
2467 TCP_SET_TIMER_MODE(mode, i);
2468 if (lo_timer == 0 || tp->t_timer[i] < lo_timer) {
2469 lo_timer = tp->t_timer[i];
2470 lo_index = i;
2471 }
2472 }
2473 }
2474 tp->tentry.index = lo_index;
2475 tp->tentry.mode = mode;
2476 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2477
2478 if (tp->tentry.index != TCPT_NONE) {
2479 tp->tentry.runtime = tp->tentry.timer_start
2480 + tp->t_timer[tp->tentry.index];
2481 if (tp->tentry.runtime == 0) {
2482 tp->tentry.runtime++;
2483 }
2484 }
2485 }
2486
2487 void
tcp_check_timer_state(struct tcpcb * tp)2488 tcp_check_timer_state(struct tcpcb *tp)
2489 {
2490 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
2491
2492 if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) {
2493 return;
2494 }
2495
2496 tcp_set_lotimer_index(tp);
2497
2498 tcp_sched_timers(tp);
2499 return;
2500 }
2501
2502 static inline void
tcp_cumulative_stat(u_int32_t cur,u_int32_t * prev,u_int32_t * dest)2503 tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest)
2504 {
2505 /* handle wrap around */
2506 int32_t diff = (int32_t) (cur - *prev);
2507 if (diff > 0) {
2508 *dest = diff;
2509 } else {
2510 *dest = 0;
2511 }
2512 *prev = cur;
2513 return;
2514 }
2515
2516 static inline void
tcp_cumulative_stat64(u_int64_t cur,u_int64_t * prev,u_int64_t * dest)2517 tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest)
2518 {
2519 /* handle wrap around */
2520 int64_t diff = (int64_t) (cur - *prev);
2521 if (diff > 0) {
2522 *dest = diff;
2523 } else {
2524 *dest = 0;
2525 }
2526 *prev = cur;
2527 return;
2528 }
2529
2530 __private_extern__ void
tcp_report_stats(void)2531 tcp_report_stats(void)
2532 {
2533 struct nstat_sysinfo_data data;
2534 struct sockaddr_in dst;
2535 struct sockaddr_in6 dst6;
2536 struct rtentry *rt = NULL;
2537 static struct tcp_last_report_stats prev;
2538 u_int64_t var, uptime;
2539
2540 #define stat data.u.tcp_stats
2541 if (((uptime = net_uptime()) - tcp_last_report_time) <
2542 tcp_report_stats_interval) {
2543 return;
2544 }
2545
2546 tcp_last_report_time = uptime;
2547
2548 bzero(&data, sizeof(data));
2549 data.flags = NSTAT_SYSINFO_TCP_STATS;
2550
2551 SOCKADDR_ZERO(&dst, sizeof(dst));
2552 dst.sin_len = sizeof(dst);
2553 dst.sin_family = AF_INET;
2554
2555 /* ipv4 avg rtt */
2556 lck_mtx_lock(rnh_lock);
2557 rt = rt_lookup(TRUE, SA(&dst), NULL,
2558 rt_tables[AF_INET], IFSCOPE_NONE);
2559 lck_mtx_unlock(rnh_lock);
2560 if (rt != NULL) {
2561 RT_LOCK(rt);
2562 if (rt_primary_default(rt, rt_key(rt)) &&
2563 rt->rt_stats != NULL) {
2564 stat.ipv4_avgrtt = rt->rt_stats->nstat_avg_rtt;
2565 }
2566 RT_UNLOCK(rt);
2567 rtfree(rt);
2568 rt = NULL;
2569 }
2570
2571 /* ipv6 avg rtt */
2572 SOCKADDR_ZERO(&dst6, sizeof(dst6));
2573 dst6.sin6_len = sizeof(dst6);
2574 dst6.sin6_family = AF_INET6;
2575
2576 lck_mtx_lock(rnh_lock);
2577 rt = rt_lookup(TRUE, SA(&dst6), NULL,
2578 rt_tables[AF_INET6], IFSCOPE_NONE);
2579 lck_mtx_unlock(rnh_lock);
2580 if (rt != NULL) {
2581 RT_LOCK(rt);
2582 if (rt_primary_default(rt, rt_key(rt)) &&
2583 rt->rt_stats != NULL) {
2584 stat.ipv6_avgrtt = rt->rt_stats->nstat_avg_rtt;
2585 }
2586 RT_UNLOCK(rt);
2587 rtfree(rt);
2588 rt = NULL;
2589 }
2590
2591 /* send packet loss rate, shift by 10 for precision */
2592 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_sndrexmitpack > 0) {
2593 var = tcpstat.tcps_sndrexmitpack << 10;
2594 stat.send_plr = (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2595 }
2596
2597 /* recv packet loss rate, shift by 10 for precision */
2598 if (tcpstat.tcps_rcvpack > 0 && tcpstat.tcps_recovered_pkts > 0) {
2599 var = tcpstat.tcps_recovered_pkts << 10;
2600 stat.recv_plr = (uint32_t)((var * 100) / tcpstat.tcps_rcvpack);
2601 }
2602
2603 /* RTO after tail loss, shift by 10 for precision */
2604 if (tcpstat.tcps_sndrexmitpack > 0
2605 && tcpstat.tcps_tailloss_rto > 0) {
2606 var = tcpstat.tcps_tailloss_rto << 10;
2607 stat.send_tlrto_rate =
2608 (uint32_t)((var * 100) / tcpstat.tcps_sndrexmitpack);
2609 }
2610
2611 /* packet reordering */
2612 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) {
2613 var = tcpstat.tcps_reordered_pkts << 10;
2614 stat.send_reorder_rate =
2615 (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2616 }
2617
2618 if (tcp_ecn_outbound == 1) {
2619 stat.ecn_client_enabled = 1;
2620 }
2621 if (tcp_ecn_inbound == 1) {
2622 stat.ecn_server_enabled = 1;
2623 }
2624 tcp_cumulative_stat(tcpstat.tcps_connattempt,
2625 &prev.tcps_connattempt, &stat.connection_attempts);
2626 tcp_cumulative_stat(tcpstat.tcps_accepts,
2627 &prev.tcps_accepts, &stat.connection_accepts);
2628 tcp_cumulative_stat(tcpstat.tcps_ecn_client_setup,
2629 &prev.tcps_ecn_client_setup, &stat.ecn_client_setup);
2630 tcp_cumulative_stat(tcpstat.tcps_ecn_server_setup,
2631 &prev.tcps_ecn_server_setup, &stat.ecn_server_setup);
2632 tcp_cumulative_stat(tcpstat.tcps_ecn_client_success,
2633 &prev.tcps_ecn_client_success, &stat.ecn_client_success);
2634 tcp_cumulative_stat(tcpstat.tcps_ecn_server_success,
2635 &prev.tcps_ecn_server_success, &stat.ecn_server_success);
2636 tcp_cumulative_stat(tcpstat.tcps_ecn_not_supported,
2637 &prev.tcps_ecn_not_supported, &stat.ecn_not_supported);
2638 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_syn,
2639 &prev.tcps_ecn_lost_syn, &stat.ecn_lost_syn);
2640 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_synack,
2641 &prev.tcps_ecn_lost_synack, &stat.ecn_lost_synack);
2642 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ce,
2643 &prev.tcps_ecn_recv_ce, &stat.ecn_recv_ce);
2644 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2645 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2646 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2647 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2648 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2649 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2650 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2651 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2652 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ce,
2653 &prev.tcps_ecn_conn_recv_ce, &stat.ecn_conn_recv_ce);
2654 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ece,
2655 &prev.tcps_ecn_conn_recv_ece, &stat.ecn_conn_recv_ece);
2656 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_plnoce,
2657 &prev.tcps_ecn_conn_plnoce, &stat.ecn_conn_plnoce);
2658 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_pl_ce,
2659 &prev.tcps_ecn_conn_pl_ce, &stat.ecn_conn_pl_ce);
2660 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_nopl_ce,
2661 &prev.tcps_ecn_conn_nopl_ce, &stat.ecn_conn_nopl_ce);
2662 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_synloss,
2663 &prev.tcps_ecn_fallback_synloss, &stat.ecn_fallback_synloss);
2664 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_reorder,
2665 &prev.tcps_ecn_fallback_reorder, &stat.ecn_fallback_reorder);
2666 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_ce,
2667 &prev.tcps_ecn_fallback_ce, &stat.ecn_fallback_ce);
2668 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_rcv,
2669 &prev.tcps_tfo_syn_data_rcv, &stat.tfo_syn_data_rcv);
2670 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req_rcv,
2671 &prev.tcps_tfo_cookie_req_rcv, &stat.tfo_cookie_req_rcv);
2672 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_sent,
2673 &prev.tcps_tfo_cookie_sent, &stat.tfo_cookie_sent);
2674 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_invalid,
2675 &prev.tcps_tfo_cookie_invalid, &stat.tfo_cookie_invalid);
2676 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req,
2677 &prev.tcps_tfo_cookie_req, &stat.tfo_cookie_req);
2678 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_rcv,
2679 &prev.tcps_tfo_cookie_rcv, &stat.tfo_cookie_rcv);
2680 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_sent,
2681 &prev.tcps_tfo_syn_data_sent, &stat.tfo_syn_data_sent);
2682 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_acked,
2683 &prev.tcps_tfo_syn_data_acked, &stat.tfo_syn_data_acked);
2684 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_loss,
2685 &prev.tcps_tfo_syn_loss, &stat.tfo_syn_loss);
2686 tcp_cumulative_stat(tcpstat.tcps_tfo_blackhole,
2687 &prev.tcps_tfo_blackhole, &stat.tfo_blackhole);
2688 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_wrong,
2689 &prev.tcps_tfo_cookie_wrong, &stat.tfo_cookie_wrong);
2690 tcp_cumulative_stat(tcpstat.tcps_tfo_no_cookie_rcv,
2691 &prev.tcps_tfo_no_cookie_rcv, &stat.tfo_no_cookie_rcv);
2692 tcp_cumulative_stat(tcpstat.tcps_tfo_heuristics_disable,
2693 &prev.tcps_tfo_heuristics_disable, &stat.tfo_heuristics_disable);
2694 tcp_cumulative_stat(tcpstat.tcps_tfo_sndblackhole,
2695 &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole);
2696
2697
2698 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt,
2699 &prev.tcps_mptcp_handover_attempt, &stat.mptcp_handover_attempt);
2700 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt,
2701 &prev.tcps_mptcp_interactive_attempt, &stat.mptcp_interactive_attempt);
2702 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt,
2703 &prev.tcps_mptcp_aggregate_attempt, &stat.mptcp_aggregate_attempt);
2704 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt,
2705 &prev.tcps_mptcp_fp_handover_attempt, &stat.mptcp_fp_handover_attempt);
2706 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt,
2707 &prev.tcps_mptcp_fp_interactive_attempt, &stat.mptcp_fp_interactive_attempt);
2708 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt,
2709 &prev.tcps_mptcp_fp_aggregate_attempt, &stat.mptcp_fp_aggregate_attempt);
2710 tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback,
2711 &prev.tcps_mptcp_heuristic_fallback, &stat.mptcp_heuristic_fallback);
2712 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback,
2713 &prev.tcps_mptcp_fp_heuristic_fallback, &stat.mptcp_fp_heuristic_fallback);
2714 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi,
2715 &prev.tcps_mptcp_handover_success_wifi, &stat.mptcp_handover_success_wifi);
2716 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell,
2717 &prev.tcps_mptcp_handover_success_cell, &stat.mptcp_handover_success_cell);
2718 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success,
2719 &prev.tcps_mptcp_interactive_success, &stat.mptcp_interactive_success);
2720 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success,
2721 &prev.tcps_mptcp_aggregate_success, &stat.mptcp_aggregate_success);
2722 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi,
2723 &prev.tcps_mptcp_fp_handover_success_wifi, &stat.mptcp_fp_handover_success_wifi);
2724 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell,
2725 &prev.tcps_mptcp_fp_handover_success_cell, &stat.mptcp_fp_handover_success_cell);
2726 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success,
2727 &prev.tcps_mptcp_fp_interactive_success, &stat.mptcp_fp_interactive_success);
2728 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success,
2729 &prev.tcps_mptcp_fp_aggregate_success, &stat.mptcp_fp_aggregate_success);
2730 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi,
2731 &prev.tcps_mptcp_handover_cell_from_wifi, &stat.mptcp_handover_cell_from_wifi);
2732 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell,
2733 &prev.tcps_mptcp_handover_wifi_from_cell, &stat.mptcp_handover_wifi_from_cell);
2734 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi,
2735 &prev.tcps_mptcp_interactive_cell_from_wifi, &stat.mptcp_interactive_cell_from_wifi);
2736 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes,
2737 &prev.tcps_mptcp_handover_cell_bytes, &stat.mptcp_handover_cell_bytes);
2738 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes,
2739 &prev.tcps_mptcp_interactive_cell_bytes, &stat.mptcp_interactive_cell_bytes);
2740 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes,
2741 &prev.tcps_mptcp_aggregate_cell_bytes, &stat.mptcp_aggregate_cell_bytes);
2742 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes,
2743 &prev.tcps_mptcp_handover_all_bytes, &stat.mptcp_handover_all_bytes);
2744 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes,
2745 &prev.tcps_mptcp_interactive_all_bytes, &stat.mptcp_interactive_all_bytes);
2746 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes,
2747 &prev.tcps_mptcp_aggregate_all_bytes, &stat.mptcp_aggregate_all_bytes);
2748 tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi,
2749 &prev.tcps_mptcp_back_to_wifi, &stat.mptcp_back_to_wifi);
2750 tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy,
2751 &prev.tcps_mptcp_wifi_proxy, &stat.mptcp_wifi_proxy);
2752 tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy,
2753 &prev.tcps_mptcp_cell_proxy, &stat.mptcp_cell_proxy);
2754 tcp_cumulative_stat(tcpstat.tcps_mptcp_triggered_cell,
2755 &prev.tcps_mptcp_triggered_cell, &stat.mptcp_triggered_cell);
2756
2757 nstat_sysinfo_send_data(&data);
2758
2759 #undef stat
2760 }
2761
2762 void
tcp_interface_send_probe(u_int16_t probe_if_index)2763 tcp_interface_send_probe(u_int16_t probe_if_index)
2764 {
2765 int32_t offset = 0;
2766 struct tcptimerlist *listp = &tcp_timer_list;
2767
2768 /* Make sure TCP clock is up to date */
2769 calculate_tcp_clock();
2770
2771 lck_mtx_lock(&listp->mtx);
2772 if (listp->probe_if_index > 0 && listp->probe_if_index != probe_if_index) {
2773 tcpstat.tcps_probe_if_conflict++;
2774 os_log(OS_LOG_DEFAULT,
2775 "%s: probe_if_index %u conflicts with %u, tcps_probe_if_conflict %u\n",
2776 __func__, probe_if_index, listp->probe_if_index,
2777 tcpstat.tcps_probe_if_conflict);
2778 goto done;
2779 }
2780
2781 listp->probe_if_index = probe_if_index;
2782 if (listp->running) {
2783 os_log(OS_LOG_DEFAULT, "%s: timer list already running for if_index %u\n",
2784 __func__, probe_if_index);
2785 goto done;
2786 }
2787
2788 /*
2789 * Reschedule the timerlist to run within the next 10ms, which is
2790 * the fastest that we can do.
2791 */
2792 offset = TCP_TIMER_10MS_QUANTUM;
2793 if (listp->scheduled) {
2794 int32_t diff;
2795 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2796 if (diff <= 0) {
2797 /* The timer will fire sooner than what's needed */
2798 os_log(OS_LOG_DEFAULT,
2799 "%s: timer will fire sooner than needed for if_index %u\n",
2800 __func__, probe_if_index);
2801 goto done;
2802 }
2803 }
2804 listp->mode = TCP_TIMERLIST_10MS_MODE;
2805 listp->idleruns = 0;
2806
2807 tcp_sched_timerlist(offset);
2808
2809 done:
2810 lck_mtx_unlock(&listp->mtx);
2811 return;
2812 }
2813
2814 /*
2815 * Enable read probes on this connection, if:
2816 * - it is in established state
2817 * - doesn't have any data outstanding
2818 * - the outgoing ifp matches
2819 * - we have not already sent any read probes
2820 */
2821 static void
tcp_enable_read_probe(struct tcpcb * tp,struct ifnet * ifp)2822 tcp_enable_read_probe(struct tcpcb *tp, struct ifnet *ifp)
2823 {
2824 if (tp->t_state == TCPS_ESTABLISHED &&
2825 tp->snd_max == tp->snd_una &&
2826 tp->t_inpcb->inp_last_outifp == ifp &&
2827 !(tp->t_flagsext & TF_DETECT_READSTALL) &&
2828 tp->t_rtimo_probes == 0) {
2829 tp->t_flagsext |= TF_DETECT_READSTALL;
2830 tp->t_rtimo_probes = 0;
2831 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
2832 TCP_TIMER_10MS_QUANTUM);
2833 if (tp->tentry.index == TCPT_NONE) {
2834 tp->tentry.index = TCPT_KEEP;
2835 tp->tentry.runtime = tcp_now +
2836 TCP_TIMER_10MS_QUANTUM;
2837 } else {
2838 int32_t diff = 0;
2839
2840 /* Reset runtime to be in next 10ms */
2841 diff = timer_diff(tp->tentry.runtime, 0,
2842 tcp_now, TCP_TIMER_10MS_QUANTUM);
2843 if (diff > 0) {
2844 tp->tentry.index = TCPT_KEEP;
2845 tp->tentry.runtime = tcp_now +
2846 TCP_TIMER_10MS_QUANTUM;
2847 if (tp->tentry.runtime == 0) {
2848 tp->tentry.runtime++;
2849 }
2850 }
2851 }
2852 }
2853 }
2854
2855 /*
2856 * Disable read probe and reset the keep alive timer
2857 */
2858 static void
tcp_disable_read_probe(struct tcpcb * tp)2859 tcp_disable_read_probe(struct tcpcb *tp)
2860 {
2861 if (tp->t_adaptive_rtimo == 0 &&
2862 ((tp->t_flagsext & TF_DETECT_READSTALL) ||
2863 tp->t_rtimo_probes > 0)) {
2864 tcp_keepalive_reset(tp);
2865
2866 if (tp->t_mpsub) {
2867 mptcp_reset_keepalive(tp);
2868 }
2869 }
2870 }
2871
2872 /*
2873 * Reschedule the tcp timerlist in the next 10ms to re-enable read/write
2874 * probes on connections going over a particular interface.
2875 */
2876 void
tcp_probe_connectivity(struct ifnet * ifp,u_int32_t enable)2877 tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable)
2878 {
2879 int32_t offset;
2880 struct tcptimerlist *listp = &tcp_timer_list;
2881 struct inpcbinfo *pcbinfo = &tcbinfo;
2882 struct inpcb *inp, *nxt;
2883
2884 if (ifp == NULL) {
2885 return;
2886 }
2887
2888 /* update clock */
2889 calculate_tcp_clock();
2890
2891 /*
2892 * Enable keep alive timer on all connections that are
2893 * active/established on this interface.
2894 */
2895 lck_rw_lock_shared(&pcbinfo->ipi_lock);
2896
2897 LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, nxt) {
2898 struct tcpcb *tp = NULL;
2899 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) ==
2900 WNT_STOPUSING) {
2901 continue;
2902 }
2903
2904 /* Acquire lock to look at the state of the connection */
2905 socket_lock(inp->inp_socket, 1);
2906
2907 /* Release the want count */
2908 if (inp->inp_ppcb == NULL ||
2909 (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) {
2910 socket_unlock(inp->inp_socket, 1);
2911 continue;
2912 }
2913 tp = intotcpcb(inp);
2914 if (enable) {
2915 tcp_enable_read_probe(tp, ifp);
2916 } else {
2917 tcp_disable_read_probe(tp);
2918 }
2919
2920 socket_unlock(inp->inp_socket, 1);
2921 }
2922 lck_rw_done(&pcbinfo->ipi_lock);
2923
2924 lck_mtx_lock(&listp->mtx);
2925 if (listp->running) {
2926 listp->pref_mode |= TCP_TIMERLIST_10MS_MODE;
2927 goto done;
2928 }
2929
2930 /* Reschedule within the next 10ms */
2931 offset = TCP_TIMER_10MS_QUANTUM;
2932 if (listp->scheduled) {
2933 int32_t diff;
2934 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2935 if (diff <= 0) {
2936 /* The timer will fire sooner than what's needed */
2937 goto done;
2938 }
2939 }
2940 listp->mode = TCP_TIMERLIST_10MS_MODE;
2941 listp->idleruns = 0;
2942
2943 tcp_sched_timerlist(offset);
2944 done:
2945 lck_mtx_unlock(&listp->mtx);
2946 return;
2947 }
2948
2949 inline void
tcp_update_mss_core(struct tcpcb * tp,struct ifnet * ifp)2950 tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp)
2951 {
2952 struct if_cellular_status_v1 *ifsr;
2953 u_int32_t optlen;
2954 ifsr = &ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
2955 if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) {
2956 optlen = tp->t_maxopd - tp->t_maxseg;
2957
2958 if (ifsr->mss_recommended ==
2959 IF_CELL_UL_MSS_RECOMMENDED_NONE &&
2960 tp->t_cached_maxopd > 0 &&
2961 tp->t_maxopd < tp->t_cached_maxopd) {
2962 tp->t_maxopd = tp->t_cached_maxopd;
2963 tcpstat.tcps_mss_to_default++;
2964 } else if (ifsr->mss_recommended ==
2965 IF_CELL_UL_MSS_RECOMMENDED_MEDIUM &&
2966 tp->t_maxopd > tcp_mss_rec_medium) {
2967 tp->t_cached_maxopd = tp->t_maxopd;
2968 tp->t_maxopd = tcp_mss_rec_medium;
2969 tcpstat.tcps_mss_to_medium++;
2970 } else if (ifsr->mss_recommended ==
2971 IF_CELL_UL_MSS_RECOMMENDED_LOW &&
2972 tp->t_maxopd > tcp_mss_rec_low) {
2973 tp->t_cached_maxopd = tp->t_maxopd;
2974 tp->t_maxopd = tcp_mss_rec_low;
2975 tcpstat.tcps_mss_to_low++;
2976 }
2977 tp->t_maxseg = tp->t_maxopd - optlen;
2978
2979 /*
2980 * clear the cached value if it is same as the current
2981 */
2982 if (tp->t_maxopd == tp->t_cached_maxopd) {
2983 tp->t_cached_maxopd = 0;
2984 }
2985 }
2986 }
2987
2988 void
tcp_update_mss_locked(struct socket * so,struct ifnet * ifp)2989 tcp_update_mss_locked(struct socket *so, struct ifnet *ifp)
2990 {
2991 struct inpcb *inp = sotoinpcb(so);
2992 struct tcpcb *tp = intotcpcb(inp);
2993
2994 if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) {
2995 return;
2996 }
2997
2998 if (!IFNET_IS_CELLULAR(ifp)) {
2999 /*
3000 * This optimization is implemented for cellular
3001 * networks only
3002 */
3003 return;
3004 }
3005 if (tp->t_state <= TCPS_CLOSE_WAIT) {
3006 /*
3007 * If the connection is currently doing or has done PMTU
3008 * blackhole detection, do not change the MSS
3009 */
3010 if (tp->t_flags & TF_BLACKHOLE) {
3011 return;
3012 }
3013 if (ifp->if_link_status == NULL) {
3014 return;
3015 }
3016 tcp_update_mss_core(tp, ifp);
3017 }
3018 }
3019
3020 void
tcp_itimer(struct inpcbinfo * ipi)3021 tcp_itimer(struct inpcbinfo *ipi)
3022 {
3023 struct inpcb *inp, *nxt;
3024
3025 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
3026 if (tcp_itimer_done == TRUE) {
3027 tcp_itimer_done = FALSE;
3028 os_atomic_inc(&ipi->ipi_timer_req.intimer_fast, relaxed);
3029 return;
3030 }
3031 /* Upgrade failed, lost lock now take it again exclusive */
3032 lck_rw_lock_exclusive(&ipi->ipi_lock);
3033 }
3034 tcp_itimer_done = TRUE;
3035
3036 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
3037 struct socket *so;
3038 struct ifnet *ifp;
3039
3040 if (inp->inp_ppcb == NULL ||
3041 in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) {
3042 continue;
3043 }
3044 so = inp->inp_socket;
3045 ifp = inp->inp_last_outifp;
3046 socket_lock(so, 1);
3047 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
3048 socket_unlock(so, 1);
3049 continue;
3050 }
3051 so_check_extended_bk_idle_time(so);
3052 if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) {
3053 tcp_update_mss_locked(so, NULL);
3054 }
3055 socket_unlock(so, 1);
3056
3057 /*
3058 * Defunct all system-initiated background sockets if the
3059 * socket is using the cellular interface and the interface
3060 * has its LQM set to abort.
3061 */
3062 if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) &&
3063 IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) &&
3064 ifp != NULL && IFNET_IS_CELLULAR(ifp) &&
3065 (ifp->if_interface_state.valid_bitmask &
3066 IF_INTERFACE_STATE_LQM_STATE_VALID) &&
3067 ifp->if_interface_state.lqm_state ==
3068 IFNET_LQM_THRESH_ABORT) {
3069 socket_defunct(current_proc(), so,
3070 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL);
3071 }
3072 }
3073
3074 ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT);
3075 lck_rw_done(&ipi->ipi_lock);
3076 }
3077
3078 void
tcp_set_link_heur_rtomin(struct tcpcb * tp,ifnet_t ifp)3079 tcp_set_link_heur_rtomin(struct tcpcb *tp, ifnet_t ifp)
3080 {
3081 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_RTOMIN) != 0 &&
3082 ifp != NULL && if_link_heuristics_enabled(ifp)) {
3083 if (tp->t_rxtcur < tcp_link_heuristics_rto_min) {
3084 IF_TCP_STATINC(ifp, linkheur_rxmtfloor);
3085 tp->t_rxtcur = tcp_link_heuristics_rto_min;
3086 }
3087 }
3088 }
3089