1 /*
2 * Copyright (c) 2000-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_timer.c 8.2 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_timer.c,v 1.34.2.11 2001/08/22 00:59:12 silby Exp $
62 */
63
64 #include "tcp_includes.h"
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/mbuf.h>
70 #include <sys/sysctl.h>
71 #include <sys/socket.h>
72 #include <sys/socketvar.h>
73 #include <sys/protosw.h>
74 #include <sys/domain.h>
75 #include <sys/mcache.h>
76 #include <sys/queue.h>
77 #include <kern/locks.h>
78 #include <kern/cpu_number.h> /* before tcp_seq.h, for tcp_random18() */
79 #include <mach/boolean.h>
80
81 #include <net/route.h>
82 #include <net/if_var.h>
83 #include <net/ntstat.h>
84
85 #include <netinet/in.h>
86 #include <netinet/in_systm.h>
87 #include <netinet/in_pcb.h>
88 #include <netinet/in_var.h>
89 #include <netinet6/in6_pcb.h>
90 #include <netinet/ip_var.h>
91 #include <netinet/tcp.h>
92 #include <netinet/tcp_cache.h>
93 #include <netinet/tcp_fsm.h>
94 #include <netinet/tcp_seq.h>
95 #include <netinet/tcp_timer.h>
96 #include <netinet/tcp_var.h>
97 #include <netinet/tcp_cc.h>
98 #include <netinet6/tcp6_var.h>
99 #include <netinet/tcpip.h>
100 #include <netinet/tcp_log.h>
101
102 #include <sys/kdebug.h>
103 #include <mach/sdt.h>
104 #include <netinet/mptcp_var.h>
105 #include <net/content_filter.h>
106 #include <net/sockaddr_utils.h>
107
108 /*
109 * If the host processor has been sleeping for too long, this is the threshold
110 * used to avoid sending stale retransmissions.
111 */
112 #define TCP_SLEEP_TOO_LONG (10 * 60 * 1000) /* 10 minutes in ms */
113
114 /* tcp timer list */
115 struct tcptimerlist tcp_timer_list;
116
117 /* List of pcbs in timewait state, protected by tcbinfo's ipi_lock */
118 struct tcptailq tcp_tw_tailq;
119
120
121 static int
122 sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
123 {
124 #pragma unused(arg2)
125 int error, temp;
126 long s, tt;
127
128 tt = *(int *)arg1;
129 s = tt * 1000 / TCP_RETRANSHZ;
130 if (tt < 0 || s > INT_MAX) {
131 return EINVAL;
132 }
133 temp = (int)s;
134
135 error = sysctl_handle_int(oidp, &temp, 0, req);
136 if (error || !req->newptr) {
137 return error;
138 }
139
140 tt = (long)temp * TCP_RETRANSHZ / 1000;
141 if (tt < 1 || tt > INT_MAX) {
142 return EINVAL;
143 }
144
145 *(int *)arg1 = (int)tt;
146 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1);
147 return 0;
148 }
149
150 #if SYSCTL_SKMEM
151 int tcp_keepinit = TCPTV_KEEP_INIT;
152 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
153 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
154 &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit),
155 sysctl_msec_to_ticks, "I", "");
156
157 int tcp_keepidle = TCPTV_KEEP_IDLE;
158 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
159 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
160 &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle),
161 sysctl_msec_to_ticks, "I", "");
162
163 int tcp_keepintvl = TCPTV_KEEPINTVL;
164 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
165 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
166 &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl),
167 sysctl_msec_to_ticks, "I", "");
168
169 SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt,
170 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
171 int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive");
172
173 int tcp_msl = TCPTV_MSL;
174 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
175 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
176 &tcp_msl, offsetof(skmem_sysctl, tcp.msl),
177 sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
178 #else /* SYSCTL_SKMEM */
179 int tcp_keepinit;
180 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
181 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
182 &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", "");
183
184 int tcp_keepidle;
185 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
186 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
187 &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", "");
188
189 int tcp_keepintvl;
190 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
191 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
192 &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
193
194 int tcp_keepcnt;
195 SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt,
196 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
197 &tcp_keepcnt, 0, "number of times to repeat keepalive");
198
199 int tcp_msl;
200 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
201 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
202 &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
203 #endif /* SYSCTL_SKMEM */
204
205 /*
206 * Avoid DoS with connections half-closed in TIME_WAIT_2
207 */
208 int tcp_fin_timeout = TCPTV_FINWAIT2;
209
210 static int
211 sysctl_tcp_fin_timeout SYSCTL_HANDLER_ARGS
212 {
213 #pragma unused(arg2)
214 int error;
215 int value = tcp_fin_timeout;
216
217 error = sysctl_handle_int(oidp, &value, 0, req);
218 if (error != 0 || req->newptr == USER_ADDR_NULL) {
219 return error;
220 }
221
222 if (value == -1) {
223 /* Reset to default value */
224 value = TCPTV_FINWAIT2;
225 } else {
226 /* Convert from milliseconds */
227 long big_value = value * TCP_RETRANSHZ / 1000;
228
229 if (big_value < 0 || big_value > INT_MAX) {
230 return EINVAL;
231 }
232 value = (int)big_value;
233 }
234 tcp_fin_timeout = value;
235 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, value);
236 return 0;
237 }
238
239 #if SYSCTL_SKMEM
240 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
241 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
242 &tcp_fin_timeout, offsetof(skmem_sysctl, tcp.fin_timeout),
243 sysctl_tcp_fin_timeout, "I", "");
244 #else /* SYSCTL_SKMEM */
245 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
246 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
247 &tcp_fin_timeout, 0,
248 sysctl_tcp_fin_timeout, "I", "");
249 #endif /* SYSCTL_SKMEM */
250
251 /*
252 * Avoid DoS via TCP Robustness in Persist Condition
253 * (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt)
254 * by allowing a system wide maximum persistence timeout value when in
255 * Zero Window Probe mode.
256 *
257 * Expressed in milliseconds to be consistent without timeout related
258 * values, the TCP socket option is in seconds.
259 */
260 #if SYSCTL_SKMEM
261 u_int32_t tcp_max_persist_timeout = 0;
262 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
263 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
264 &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout),
265 sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP");
266 #else /* SYSCTL_SKMEM */
267 u_int32_t tcp_max_persist_timeout = 0;
268 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
269 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
270 &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I",
271 "Maximum persistence timeout for ZWP");
272 #endif /* SYSCTL_SKMEM */
273
274 SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive,
275 CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0,
276 "Assume SO_KEEPALIVE on all TCP connections");
277
278 /*
279 * This parameter determines how long the timer list will stay in fast or
280 * quick mode even though all connections are idle. In this state, the
281 * timer will run more frequently anticipating new data.
282 */
283 SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax,
284 CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax,
285 TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode");
286
287 /*
288 * See tcp_syn_backoff[] for interval values between SYN retransmits;
289 * the value set below defines the number of retransmits, before we
290 * disable the timestamp and window scaling options during subsequent
291 * SYN retransmits. Setting it to 0 disables the dropping off of those
292 * two options.
293 */
294 SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres,
295 CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres,
296 10, "Number of retransmitted SYNs before disabling RFC 1323 "
297 "options on local connections");
298
299 static int tcp_timer_advanced = 0;
300 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced,
301 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_timer_advanced, 0,
302 "Number of times one of the timers was advanced");
303
304 static int tcp_resched_timerlist = 0;
305 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist,
306 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0,
307 "Number of times timer list was rescheduled as part of processing a packet");
308
309 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection,
310 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1,
311 "Path MTU Discovery Black Hole Detection");
312
313 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss,
314 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200,
315 "Path MTU Discovery Black Hole Detection lowered MSS");
316
317 #if (DEBUG || DEVELOPMENT)
318 int tcp_probe_if_fix_port = 0;
319 SYSCTL_INT(_net_inet_tcp, OID_AUTO, probe_if_fix_port,
320 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
321 &tcp_probe_if_fix_port, 0, "");
322 #endif /* (DEBUG || DEVELOPMENT) */
323
324 static u_int32_t tcp_mss_rec_medium = 1200;
325 static u_int32_t tcp_mss_rec_low = 512;
326
327 #define TCP_REPORT_STATS_INTERVAL 43200 /* 12 hours, in seconds */
328 int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL;
329
330 /* performed garbage collection of "used" sockets */
331 static boolean_t tcp_gc_done = FALSE;
332
333 /* max idle probes */
334 int tcp_maxpersistidle = TCPTV_KEEP_IDLE;
335
336 /*
337 * TCP delack timer is set to 100 ms. Since the processing of timer list
338 * in fast mode will happen no faster than 100 ms, the delayed ack timer
339 * will fire some where between 100 and 200 ms.
340 */
341 int tcp_delack = TCP_RETRANSHZ / 10;
342
343 #if MPTCP
344 /*
345 * MP_JOIN retransmission of 3rd ACK will be every 500 msecs without backoff
346 */
347 int tcp_jack_rxmt = TCP_RETRANSHZ / 2;
348 #endif /* MPTCP */
349
350 static boolean_t tcp_itimer_done = FALSE;
351
352 static void tcp_remove_timer(struct tcpcb *tp);
353 static void tcp_sched_timerlist(uint32_t offset);
354 static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode,
355 u_int16_t probe_if_index);
356 static inline void tcp_set_lotimer_index(struct tcpcb *);
357 __private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp);
358 static inline void tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp);
359 __private_extern__ void tcp_report_stats(void);
360
361 static u_int64_t tcp_last_report_time;
362
363 /*
364 * Structure to store previously reported stats so that we can send
365 * incremental changes in each report interval.
366 */
367 struct tcp_last_report_stats {
368 u_int32_t tcps_connattempt;
369 u_int32_t tcps_accepts;
370 u_int32_t tcps_ecn_client_setup;
371 u_int32_t tcps_ecn_server_setup;
372 u_int32_t tcps_ecn_client_success;
373 u_int32_t tcps_ecn_server_success;
374 u_int32_t tcps_ecn_not_supported;
375 u_int32_t tcps_ecn_lost_syn;
376 u_int32_t tcps_ecn_lost_synack;
377 u_int32_t tcps_ecn_recv_ce;
378 u_int32_t tcps_ecn_recv_ece;
379 u_int32_t tcps_ecn_sent_ece;
380 u_int32_t tcps_ecn_conn_recv_ce;
381 u_int32_t tcps_ecn_conn_recv_ece;
382 u_int32_t tcps_ecn_conn_plnoce;
383 u_int32_t tcps_ecn_conn_pl_ce;
384 u_int32_t tcps_ecn_conn_nopl_ce;
385 u_int32_t tcps_ecn_fallback_synloss;
386 u_int32_t tcps_ecn_fallback_reorder;
387 u_int32_t tcps_ecn_fallback_ce;
388
389 /* TFO-related statistics */
390 u_int32_t tcps_tfo_syn_data_rcv;
391 u_int32_t tcps_tfo_cookie_req_rcv;
392 u_int32_t tcps_tfo_cookie_sent;
393 u_int32_t tcps_tfo_cookie_invalid;
394 u_int32_t tcps_tfo_cookie_req;
395 u_int32_t tcps_tfo_cookie_rcv;
396 u_int32_t tcps_tfo_syn_data_sent;
397 u_int32_t tcps_tfo_syn_data_acked;
398 u_int32_t tcps_tfo_syn_loss;
399 u_int32_t tcps_tfo_blackhole;
400 u_int32_t tcps_tfo_cookie_wrong;
401 u_int32_t tcps_tfo_no_cookie_rcv;
402 u_int32_t tcps_tfo_heuristics_disable;
403 u_int32_t tcps_tfo_sndblackhole;
404
405 /* MPTCP-related statistics */
406 u_int32_t tcps_mptcp_handover_attempt;
407 u_int32_t tcps_mptcp_interactive_attempt;
408 u_int32_t tcps_mptcp_aggregate_attempt;
409 u_int32_t tcps_mptcp_fp_handover_attempt;
410 u_int32_t tcps_mptcp_fp_interactive_attempt;
411 u_int32_t tcps_mptcp_fp_aggregate_attempt;
412 u_int32_t tcps_mptcp_heuristic_fallback;
413 u_int32_t tcps_mptcp_fp_heuristic_fallback;
414 u_int32_t tcps_mptcp_handover_success_wifi;
415 u_int32_t tcps_mptcp_handover_success_cell;
416 u_int32_t tcps_mptcp_interactive_success;
417 u_int32_t tcps_mptcp_aggregate_success;
418 u_int32_t tcps_mptcp_fp_handover_success_wifi;
419 u_int32_t tcps_mptcp_fp_handover_success_cell;
420 u_int32_t tcps_mptcp_fp_interactive_success;
421 u_int32_t tcps_mptcp_fp_aggregate_success;
422 u_int32_t tcps_mptcp_handover_cell_from_wifi;
423 u_int32_t tcps_mptcp_handover_wifi_from_cell;
424 u_int32_t tcps_mptcp_interactive_cell_from_wifi;
425 u_int64_t tcps_mptcp_handover_cell_bytes;
426 u_int64_t tcps_mptcp_interactive_cell_bytes;
427 u_int64_t tcps_mptcp_aggregate_cell_bytes;
428 u_int64_t tcps_mptcp_handover_all_bytes;
429 u_int64_t tcps_mptcp_interactive_all_bytes;
430 u_int64_t tcps_mptcp_aggregate_all_bytes;
431 u_int32_t tcps_mptcp_back_to_wifi;
432 u_int32_t tcps_mptcp_wifi_proxy;
433 u_int32_t tcps_mptcp_cell_proxy;
434 u_int32_t tcps_mptcp_triggered_cell;
435 };
436
437
438 /* Returns true if the timer is on the timer list */
439 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
440
441 /* Run the TCP timerlist atleast once every hour */
442 #define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ)
443
444
445 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
446 static boolean_t tcp_garbage_collect(struct inpcb *, int);
447
448 #define TIMERENTRY_TO_TP(te) (__unsafe_forge_single(struct tcpcb *, ((uintptr_t)te - offsetof(struct tcpcb, tentry.te_le.le_next))))
449
450 #define VERIFY_NEXT_LINK(elm, field) do { \
451 if (LIST_NEXT((elm),field) != NULL && \
452 LIST_NEXT((elm),field)->field.le_prev != \
453 &((elm)->field.le_next)) \
454 panic("Bad link elm %p next->prev != elm", (elm)); \
455 } while(0)
456
457 #define VERIFY_PREV_LINK(elm, field) do { \
458 if (*(elm)->field.le_prev != (elm)) \
459 panic("Bad link elm %p prev->next != elm", (elm)); \
460 } while(0)
461
462 #define TCP_SET_TIMER_MODE(mode, i) do { \
463 if (IS_TIMER_HZ_10MS(i)) \
464 (mode) |= TCP_TIMERLIST_10MS_MODE; \
465 else if (IS_TIMER_HZ_100MS(i)) \
466 (mode) |= TCP_TIMERLIST_100MS_MODE; \
467 else \
468 (mode) |= TCP_TIMERLIST_500MS_MODE; \
469 } while(0)
470
471 #if (DEVELOPMENT || DEBUG)
472 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_medium,
473 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_medium, 0,
474 "Medium MSS based on recommendation in link status report");
475 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_low,
476 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_low, 0,
477 "Low MSS based on recommendation in link status report");
478
479 static int32_t tcp_change_mss_recommended = 0;
480 static int
481 sysctl_change_mss_recommended SYSCTL_HANDLER_ARGS
482 {
483 #pragma unused(oidp, arg1, arg2)
484 int i, err = 0, changed = 0;
485 struct ifnet *ifp;
486 struct if_link_status ifsr;
487 struct if_cellular_status_v1 *new_cell_sr;
488 err = sysctl_io_number(req, tcp_change_mss_recommended,
489 sizeof(int32_t), &i, &changed);
490 if (changed) {
491 if (i < 0 || i > UINT16_MAX) {
492 return EINVAL;
493 }
494 ifnet_head_lock_shared();
495 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
496 if (IFNET_IS_CELLULAR(ifp)) {
497 bzero(&ifsr, sizeof(ifsr));
498 new_cell_sr = &ifsr.ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
499 ifsr.ifsr_version = IF_CELLULAR_STATUS_REPORT_CURRENT_VERSION;
500 ifsr.ifsr_len = sizeof(*new_cell_sr);
501
502 /* Set MSS recommended */
503 new_cell_sr->valid_bitmask |= IF_CELL_UL_MSS_RECOMMENDED_VALID;
504 new_cell_sr->mss_recommended = (uint16_t)i;
505 err = ifnet_link_status_report(ifp, new_cell_sr, sizeof(new_cell_sr));
506 if (err == 0) {
507 tcp_change_mss_recommended = i;
508 } else {
509 break;
510 }
511 }
512 }
513 ifnet_head_done();
514 }
515 return err;
516 }
517
518 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, change_mss_recommended,
519 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_change_mss_recommended,
520 0, sysctl_change_mss_recommended, "IU", "Change MSS recommended");
521
522 SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval,
523 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0,
524 "Report stats interval");
525 #endif /* (DEVELOPMENT || DEBUG) */
526
527 /*
528 * Macro to compare two timers. If there is a reset of the sign bit,
529 * it is safe to assume that the timer has wrapped around. By doing
530 * signed comparision, we take care of wrap around such that the value
531 * with the sign bit reset is actually ahead of the other.
532 */
533 inline int32_t
timer_diff(uint32_t t1,uint32_t toff1,uint32_t t2,uint32_t toff2)534 timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2)
535 {
536 return (int32_t)((t1 + toff1) - (t2 + toff2));
537 }
538
539 /*
540 * Add to tcp timewait list, delay is given in milliseconds.
541 */
542 static void
add_to_time_wait_locked(struct tcpcb * tp,uint32_t delay)543 add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay)
544 {
545 struct inpcbinfo *pcbinfo = &tcbinfo;
546 struct inpcb *inp = tp->t_inpcb;
547 uint32_t timer;
548
549 /* pcb list should be locked when we get here */
550 LCK_RW_ASSERT(&pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE);
551
552 /* We may get here multiple times, so check */
553 if (!(inp->inp_flags2 & INP2_TIMEWAIT)) {
554 pcbinfo->ipi_twcount++;
555 inp->inp_flags2 |= INP2_TIMEWAIT;
556
557 /* Remove from global inp list */
558 LIST_REMOVE(inp, inp_list);
559 } else {
560 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
561 }
562
563 /* Compute the time at which this socket can be closed */
564 timer = tcp_now + delay;
565
566 /* We will use the TCPT_2MSL timer for tracking this delay */
567
568 if (TIMER_IS_ON_LIST(tp)) {
569 tcp_remove_timer(tp);
570 }
571 tp->t_timer[TCPT_2MSL] = timer;
572
573 TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry);
574 }
575
576 void
add_to_time_wait(struct tcpcb * tp,uint32_t delay)577 add_to_time_wait(struct tcpcb *tp, uint32_t delay)
578 {
579 if (tp->t_inpcb->inp_socket->so_options & SO_NOWAKEFROMSLEEP) {
580 socket_post_kev_msg_closed(tp->t_inpcb->inp_socket);
581 }
582
583 tcp_del_fsw_flow(tp);
584
585 /* 19182803: Notify nstat that connection is closing before waiting. */
586 nstat_pcb_detach(tp->t_inpcb);
587
588 #if CONTENT_FILTER
589 if ((tp->t_inpcb->inp_socket->so_flags & SOF_CONTENT_FILTER) != 0) {
590 /* If filter present, allow filter to finish processing all queued up data before adding to time wait queue */
591 (void) cfil_sock_tcp_add_time_wait(tp->t_inpcb->inp_socket);
592 } else
593 #endif /* CONTENT_FILTER */
594 {
595 add_to_time_wait_now(tp, delay);
596 }
597 }
598
599 void
add_to_time_wait_now(struct tcpcb * tp,uint32_t delay)600 add_to_time_wait_now(struct tcpcb *tp, uint32_t delay)
601 {
602 struct inpcbinfo *pcbinfo = &tcbinfo;
603
604 if (!lck_rw_try_lock_exclusive(&pcbinfo->ipi_lock)) {
605 socket_unlock(tp->t_inpcb->inp_socket, 0);
606 lck_rw_lock_exclusive(&pcbinfo->ipi_lock);
607 socket_lock(tp->t_inpcb->inp_socket, 0);
608 }
609 add_to_time_wait_locked(tp, delay);
610 lck_rw_done(&pcbinfo->ipi_lock);
611
612 inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY);
613 }
614
615 /* If this is on time wait queue, remove it. */
616 void
tcp_remove_from_time_wait(struct inpcb * inp)617 tcp_remove_from_time_wait(struct inpcb *inp)
618 {
619 struct tcpcb *tp = intotcpcb(inp);
620 if (inp->inp_flags2 & INP2_TIMEWAIT) {
621 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
622 }
623 }
624
625 static boolean_t
tcp_garbage_collect(struct inpcb * inp,int istimewait)626 tcp_garbage_collect(struct inpcb *inp, int istimewait)
627 {
628 boolean_t active = FALSE;
629 struct socket *so, *mp_so = NULL;
630 struct tcpcb *tp;
631
632 so = inp->inp_socket;
633 tp = intotcpcb(inp);
634
635 if (so->so_flags & SOF_MP_SUBFLOW) {
636 mp_so = mptetoso(tptomptp(tp)->mpt_mpte);
637 if (!socket_try_lock(mp_so)) {
638 mp_so = NULL;
639 active = TRUE;
640 goto out;
641 }
642 if (mpsotomppcb(mp_so)->mpp_inside > 0) {
643 os_log(mptcp_log_handle, "%s - %lx: Still inside %d usecount %d\n", __func__,
644 (unsigned long)VM_KERNEL_ADDRPERM(mpsotompte(mp_so)),
645 mpsotomppcb(mp_so)->mpp_inside,
646 mp_so->so_usecount);
647 socket_unlock(mp_so, 0);
648 mp_so = NULL;
649 active = TRUE;
650 goto out;
651 }
652 /* We call socket_unlock with refcount further below */
653 mp_so->so_usecount++;
654 tptomptp(tp)->mpt_mpte->mpte_mppcb->mpp_inside++;
655 }
656
657 /*
658 * Skip if still in use or busy; it would have been more efficient
659 * if we were to test so_usecount against 0, but this isn't possible
660 * due to the current implementation of tcp_dropdropablreq() where
661 * overflow sockets that are eligible for garbage collection have
662 * their usecounts set to 1.
663 */
664 if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) {
665 active = TRUE;
666 goto out;
667 }
668
669 /* Check again under the lock */
670 if (so->so_usecount > 1) {
671 if (inp->inp_wantcnt == WNT_STOPUSING) {
672 active = TRUE;
673 }
674 lck_mtx_unlock(&inp->inpcb_mtx);
675 goto out;
676 }
677
678 if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) &&
679 tp->t_state != TCPS_CLOSED) {
680 /* Become a regular mutex */
681 lck_mtx_convert_spin(&inp->inpcb_mtx);
682 tcp_close(tp);
683 }
684
685 /*
686 * Overflowed socket dropped from the listening queue? Do this
687 * only if we are called to clean up the time wait slots, since
688 * tcp_dropdropablreq() considers a socket to have been fully
689 * dropped after add_to_time_wait() is finished.
690 * Also handle the case of connections getting closed by the peer
691 * while in the queue as seen with rdar://6422317
692 *
693 */
694 if (so->so_usecount == 1 &&
695 ((istimewait && (so->so_flags & SOF_OVERFLOW)) ||
696 ((tp != NULL) && (tp->t_state == TCPS_CLOSED) &&
697 (so->so_head != NULL) &&
698 ((so->so_state & (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE)) ==
699 (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE))))) {
700 if (inp->inp_state != INPCB_STATE_DEAD) {
701 /* Become a regular mutex */
702 lck_mtx_convert_spin(&inp->inpcb_mtx);
703 if (SOCK_CHECK_DOM(so, PF_INET6)) {
704 in6_pcbdetach(inp);
705 } else {
706 in_pcbdetach(inp);
707 }
708 }
709 VERIFY(so->so_usecount > 0);
710 so->so_usecount--;
711 if (inp->inp_wantcnt == WNT_STOPUSING) {
712 active = TRUE;
713 }
714 lck_mtx_unlock(&inp->inpcb_mtx);
715 goto out;
716 } else if (inp->inp_wantcnt != WNT_STOPUSING) {
717 lck_mtx_unlock(&inp->inpcb_mtx);
718 active = FALSE;
719 goto out;
720 }
721
722 /*
723 * We get here because the PCB is no longer searchable
724 * (WNT_STOPUSING); detach (if needed) and dispose if it is dead
725 * (usecount is 0). This covers all cases, including overflow
726 * sockets and those that are considered as "embryonic",
727 * i.e. created by sonewconn() in TCP input path, and have
728 * not yet been committed. For the former, we reduce the usecount
729 * to 0 as done by the code above. For the latter, the usecount
730 * would have reduced to 0 as part calling soabort() when the
731 * socket is dropped at the end of tcp_input().
732 */
733 if (so->so_usecount == 0) {
734 DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp,
735 struct tcpcb *, tp, int32_t, TCPS_CLOSED);
736 /* Become a regular mutex */
737 lck_mtx_convert_spin(&inp->inpcb_mtx);
738
739 /*
740 * If this tp still happens to be on the timer list,
741 * take it out
742 */
743 if (TIMER_IS_ON_LIST(tp)) {
744 tcp_remove_timer(tp);
745 }
746
747 if (inp->inp_state != INPCB_STATE_DEAD) {
748 if (SOCK_CHECK_DOM(so, PF_INET6)) {
749 in6_pcbdetach(inp);
750 } else {
751 in_pcbdetach(inp);
752 }
753 }
754
755 if (mp_so) {
756 mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub);
757
758 /* so is now unlinked from mp_so - let's drop the lock */
759 socket_unlock(mp_so, 1);
760 mp_so = NULL;
761 }
762
763 in_pcbdispose(inp);
764 active = FALSE;
765 goto out;
766 }
767
768 lck_mtx_unlock(&inp->inpcb_mtx);
769 active = TRUE;
770
771 out:
772 if (mp_so) {
773 socket_unlock(mp_so, 1);
774 }
775
776 return active;
777 }
778
779 /*
780 * TCP garbage collector callback (inpcb_timer_func_t).
781 *
782 * Returns the number of pcbs that will need to be gc-ed soon,
783 * returnining > 0 will keep timer active.
784 */
785 void
tcp_gc(struct inpcbinfo * ipi)786 tcp_gc(struct inpcbinfo *ipi)
787 {
788 struct inpcb *inp, *nxt;
789 struct tcpcb *tw_tp, *tw_ntp;
790 #if KDEBUG
791 static int tws_checked = 0;
792 #endif
793
794 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0);
795
796 /*
797 * Update tcp_now here as it may get used while
798 * processing the slow timer.
799 */
800 calculate_tcp_clock();
801
802 /*
803 * Garbage collect socket/tcpcb: We need to acquire the list lock
804 * exclusively to do this
805 */
806
807 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
808 /* don't sweat it this time; cleanup was done last time */
809 if (tcp_gc_done == TRUE) {
810 tcp_gc_done = FALSE;
811 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END,
812 tws_checked, cur_tw_slot, 0, 0, 0);
813 /* Lock upgrade failed, give up this round */
814 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
815 return;
816 }
817 /* Upgrade failed, lost lock now take it again exclusive */
818 lck_rw_lock_exclusive(&ipi->ipi_lock);
819 }
820 tcp_gc_done = TRUE;
821
822 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
823 if (tcp_garbage_collect(inp, 0)) {
824 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
825 }
826 }
827
828 /* Now cleanup the time wait ones */
829 TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) {
830 /*
831 * We check the timestamp here without holding the
832 * socket lock for better performance. If there are
833 * any pcbs in time-wait, the timer will get rescheduled.
834 * Hence some error in this check can be tolerated.
835 *
836 * Sometimes a socket on time-wait queue can be closed if
837 * 2MSL timer expired but the application still has a
838 * usecount on it.
839 */
840 if (tw_tp->t_state == TCPS_CLOSED ||
841 TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) {
842 if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) {
843 os_atomic_inc(&ipi->ipi_gc_req.intimer_lazy, relaxed);
844 }
845 }
846 }
847
848 /* take into account pcbs that are still in time_wait_slots */
849 os_atomic_add(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount, relaxed);
850
851 lck_rw_done(&ipi->ipi_lock);
852
853 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked,
854 cur_tw_slot, 0, 0, 0);
855
856 return;
857 }
858
859 /*
860 * Cancel all timers for TCP tp.
861 */
862 void
tcp_canceltimers(struct tcpcb * tp)863 tcp_canceltimers(struct tcpcb *tp)
864 {
865 int i;
866
867 tcp_remove_timer(tp);
868 for (i = 0; i < TCPT_NTIMERS; i++) {
869 tp->t_timer[i] = 0;
870 }
871 tp->tentry.te_timer_start = tcp_now;
872 tp->tentry.te_index = TCPT_NONE;
873 }
874
875 static int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =
876 { 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 };
877
878 int tcp_backoff[TCP_MAXRXTSHIFT + 1] =
879 { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
880
881 static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */
882
883 void
tcp_rexmt_save_state(struct tcpcb * tp)884 tcp_rexmt_save_state(struct tcpcb *tp)
885 {
886 u_int32_t fsize;
887 if (TSTMP_SUPPORTED(tp)) {
888 /*
889 * Since timestamps are supported on the connection,
890 * we can do recovery as described in rfc 4015.
891 */
892 fsize = tp->snd_max - tp->snd_una;
893 tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh);
894 tp->snd_recover_prev = tp->snd_recover;
895 } else {
896 /*
897 * Timestamp option is not supported on this connection.
898 * Record ssthresh and cwnd so they can
899 * be recovered if this turns out to be a "bad" retransmit.
900 * A retransmit is considered "bad" if an ACK for this
901 * segment is received within RTT/2 interval; the assumption
902 * here is that the ACK was already in flight. See
903 * "On Estimating End-to-End Network Path Properties" by
904 * Allman and Paxson for more details.
905 */
906 tp->snd_cwnd_prev = tp->snd_cwnd;
907 tp->snd_ssthresh_prev = tp->snd_ssthresh;
908 tp->snd_recover_prev = tp->snd_recover;
909 if (IN_FASTRECOVERY(tp)) {
910 tp->t_flags |= TF_WASFRECOVERY;
911 } else {
912 tp->t_flags &= ~TF_WASFRECOVERY;
913 }
914 }
915 tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2;
916 tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT);
917 tp->t_flagsext &= ~(TF_RECOMPUTE_RTT);
918 }
919
920 /*
921 * Revert to the older segment size if there is an indication that PMTU
922 * blackhole detection was not needed.
923 */
924 void
tcp_pmtud_revert_segment_size(struct tcpcb * tp)925 tcp_pmtud_revert_segment_size(struct tcpcb *tp)
926 {
927 int32_t optlen;
928
929 VERIFY(tp->t_pmtud_saved_maxopd > 0);
930 tp->t_flags |= TF_PMTUD;
931 tp->t_flags &= ~TF_BLACKHOLE;
932 optlen = tp->t_maxopd - tp->t_maxseg;
933 tp->t_maxopd = tp->t_pmtud_saved_maxopd;
934 tp->t_maxseg = tp->t_maxopd - optlen;
935
936 /*
937 * Reset the slow-start flight size as it
938 * may depend on the new MSS
939 */
940 if (CC_ALGO(tp)->cwnd_init != NULL) {
941 CC_ALGO(tp)->cwnd_init(tp);
942 }
943
944 if (TCP_USE_RLEDBAT(tp, tp->t_inpcb->inp_socket) &&
945 tcp_cc_rledbat.rwnd_init != NULL) {
946 tcp_cc_rledbat.rwnd_init(tp);
947 }
948
949 tp->t_pmtud_start_ts = 0;
950 tcpstat.tcps_pmtudbh_reverted++;
951
952 /* change MSS according to recommendation, if there was one */
953 tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL);
954 }
955
956 static uint32_t
tcp_pmtud_black_holed_next_mss(struct tcpcb * tp)957 tcp_pmtud_black_holed_next_mss(struct tcpcb *tp)
958 {
959 /* Reduce the MSS to intermediary value */
960 if (tp->t_maxopd > tcp_pmtud_black_hole_mss) {
961 return tcp_pmtud_black_hole_mss;
962 } else {
963 if (tp->t_inpcb->inp_vflag & INP_IPV4) {
964 return tcp_mssdflt;
965 } else {
966 return tcp_v6mssdflt;
967 }
968 }
969 }
970
971 /*
972 * Send a packet designed to force a response
973 * if the peer is up and reachable:
974 * either an ACK if the connection is still alive,
975 * or an RST if the peer has closed the connection
976 * due to timeout or reboot.
977 * Using sequence number tp->snd_una-1
978 * causes the transmitted zero-length segment
979 * to lie outside the receive window;
980 * by the protocol spec, this requires the
981 * correspondent TCP to respond.
982 */
983 static bool
tcp_send_keep_alive(struct tcpcb * tp)984 tcp_send_keep_alive(struct tcpcb *tp)
985 {
986 struct tcptemp *__single t_template;
987 struct mbuf *__single m;
988
989 tcpstat.tcps_keepprobe++;
990 t_template = tcp_maketemplate(tp, &m, NULL, NULL);
991 if (t_template != NULL) {
992 struct inpcb *inp = tp->t_inpcb;
993 struct tcp_respond_args tra;
994
995 bzero(&tra, sizeof(tra));
996 tra.nocell = INP_NO_CELLULAR(inp) ? 1 : 0;
997 tra.noexpensive = INP_NO_EXPENSIVE(inp) ? 1 : 0;
998 tra.noconstrained = INP_NO_CONSTRAINED(inp) ? 1 : 0;
999 tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp) ? 1 : 0;
1000 tra.intcoproc_allowed = INP_INTCOPROC_ALLOWED(inp) ? 1 : 0;
1001 tra.management_allowed = INP_MANAGEMENT_ALLOWED(inp) ? 1 : 0;
1002 tra.keep_alive = 1;
1003 if (tp->t_inpcb->inp_flags & INP_BOUND_IF) {
1004 tra.ifscope = tp->t_inpcb->inp_boundifp->if_index;
1005 } else {
1006 tra.ifscope = IFSCOPE_NONE;
1007 }
1008 tcp_respond(tp, t_template->tt_ipgen, sizeof(t_template->tt_ipgen),
1009 &t_template->tt_t, (struct mbuf *)NULL,
1010 tp->rcv_nxt, tp->snd_una - 1, 0, 0, NULL, 0, 0, 0, &tra, false);
1011 (void) m_free(m);
1012 return true;
1013 } else {
1014 return false;
1015 }
1016 }
1017
1018 /*
1019 * TCP timer processing.
1020 */
1021 struct tcpcb *
tcp_timers(struct tcpcb * tp,int timer)1022 tcp_timers(struct tcpcb *tp, int timer)
1023 {
1024 int32_t rexmt, optlen = 0, idle_time = 0;
1025 struct socket *so;
1026 u_int64_t accsleep_ms;
1027 u_int64_t last_sleep_ms = 0;
1028 struct ifnet *outifp = tp->t_inpcb->inp_last_outifp;
1029
1030 so = tp->t_inpcb->inp_socket;
1031 idle_time = tcp_now - tp->t_rcvtime;
1032
1033 switch (timer) {
1034 /*
1035 * 2 MSL timeout in shutdown went off. If we're closed but
1036 * still waiting for peer to close and connection has been idle
1037 * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2,
1038 * delete connection control block.
1039 * Otherwise, (this case shouldn't happen) check again in a bit
1040 * we keep the socket in the main list in that case.
1041 */
1042 case TCPT_2MSL:
1043 tcp_free_sackholes(tp);
1044 if (tp->t_state != TCPS_TIME_WAIT &&
1045 tp->t_state != TCPS_FIN_WAIT_2 &&
1046 ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) {
1047 tp->t_timer[TCPT_2MSL] = tcp_offset_from_start(tp,
1048 (u_int32_t)TCP_CONN_KEEPINTVL(tp));
1049 } else {
1050 if (tp->t_state == TCPS_FIN_WAIT_2) {
1051 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1052 "FIN wait timeout drop");
1053 tcpstat.tcps_fin_timeout_drops++;
1054 tp = tcp_drop(tp, 0);
1055 } else {
1056 tp = tcp_close(tp);
1057 }
1058 return tp;
1059 }
1060 break;
1061
1062 /*
1063 * Retransmission timer went off. Message has not
1064 * been acked within retransmit interval. Back off
1065 * to a longer retransmit interval and retransmit one segment.
1066 */
1067 case TCPT_REXMT:
1068 absolutetime_to_nanoseconds(mach_absolutetime_asleep,
1069 &accsleep_ms);
1070 accsleep_ms = accsleep_ms / 1000000UL;
1071 if (accsleep_ms > tp->t_accsleep_ms) {
1072 last_sleep_ms = accsleep_ms - tp->t_accsleep_ms;
1073 }
1074 /*
1075 * Drop a connection in the retransmit timer
1076 * 1. If we have retransmitted more than TCP_MAXRXTSHIFT
1077 * times
1078 * 2. If the time spent in this retransmission episode is
1079 * more than the time limit set with TCP_RXT_CONNDROPTIME
1080 * socket option
1081 * 3. If TCP_RXT_FINDROP socket option was set and
1082 * we have already retransmitted the FIN 3 times without
1083 * receiving an ack
1084 */
1085 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT ||
1086 (tp->t_rxt_conndroptime > 0 && tp->t_rxtstart > 0 &&
1087 (tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime) ||
1088 ((tp->t_flagsext & TF_RXTFINDROP) != 0 &&
1089 (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) ||
1090 (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) {
1091 if (tp->t_state == TCPS_ESTABLISHED &&
1092 tp->t_rxt_minimum_timeout > 0) {
1093 /*
1094 * Avoid dropping a connection if minimum
1095 * timeout is set and that time did not
1096 * pass. We will retry sending
1097 * retransmissions at the maximum interval
1098 */
1099 if (TSTMP_LT(tcp_now, (tp->t_rxtstart +
1100 tp->t_rxt_minimum_timeout))) {
1101 tp->t_rxtshift = TCP_MAXRXTSHIFT - 1;
1102 goto retransmit_packet;
1103 }
1104 }
1105 if ((tp->t_flagsext & TF_RXTFINDROP) != 0) {
1106 tcpstat.tcps_rxtfindrop++;
1107 } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) {
1108 tcpstat.tcps_drop_after_sleep++;
1109 } else {
1110 tcpstat.tcps_timeoutdrop++;
1111 }
1112
1113 tp->t_rxtshift = TCP_MAXRXTSHIFT;
1114 soevent(so,
1115 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1116
1117 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1118 "retransmission timeout drop");
1119 tp = tcp_drop(tp, tp->t_softerror ?
1120 tp->t_softerror : ETIMEDOUT);
1121
1122 break;
1123 }
1124 retransmit_packet:
1125 tcpstat.tcps_rexmttimeo++;
1126 tp->t_accsleep_ms = accsleep_ms;
1127
1128 if (tp->t_rxtshift == 1 &&
1129 tp->t_state == TCPS_ESTABLISHED) {
1130 /* Set the time at which retransmission started. */
1131 tp->t_rxtstart = tcp_now;
1132
1133 /*
1134 * if this is the first retransmit timeout, save
1135 * the state so that we can recover if the timeout
1136 * is spurious.
1137 */
1138 tcp_rexmt_save_state(tp);
1139 tcp_ccdbg_trace(tp, NULL, TCP_CC_FIRST_REXMT);
1140 }
1141 #if MPTCP
1142 if ((tp->t_rxtshift >= mptcp_fail_thresh) &&
1143 (tp->t_state == TCPS_ESTABLISHED) &&
1144 (tp->t_mpflags & TMPF_MPTCP_TRUE)) {
1145 mptcp_act_on_txfail(so);
1146 }
1147
1148 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1149 (so->so_flags & SOF_MP_SUBFLOW)) {
1150 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1151
1152 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER ||
1153 mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
1154 mptcp_check_subflows_and_add(mpte);
1155 }
1156 }
1157 #endif /* MPTCP */
1158
1159 if (tp->t_adaptive_wtimo > 0 &&
1160 tp->t_rxtshift > tp->t_adaptive_wtimo &&
1161 TCPS_HAVEESTABLISHED(tp->t_state)) {
1162 /* Send an event to the application */
1163 soevent(so,
1164 (SO_FILT_HINT_LOCKED |
1165 SO_FILT_HINT_ADAPTIVE_WTIMO));
1166 }
1167
1168 /*
1169 * If this is a retransmit timeout after PTO, the PTO
1170 * was not effective
1171 */
1172 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1173 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1174 tcpstat.tcps_rto_after_pto++;
1175 }
1176
1177 if (tp->t_flagsext & TF_DELAY_RECOVERY) {
1178 /*
1179 * Retransmit timer fired before entering recovery
1180 * on a connection with packet re-ordering. This
1181 * suggests that the reordering metrics computed
1182 * are not accurate.
1183 */
1184 tp->t_reorderwin = 0;
1185 tp->t_timer[TCPT_DELAYFR] = 0;
1186 tp->t_flagsext &= ~(TF_DELAY_RECOVERY);
1187 }
1188
1189 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1190 tp->t_state == TCPS_SYN_RECEIVED) {
1191 tcp_disable_tfo(tp);
1192 }
1193
1194 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1195 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1196 (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
1197 !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
1198 ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||
1199 tp->t_rxtshift > 4)) {
1200 /*
1201 * For regular retransmissions, a first one is being
1202 * done for tail-loss probe.
1203 * Thus, if rxtshift > 1, this means we have sent the segment
1204 * a total of 3 times.
1205 *
1206 * If we are in SYN-SENT state, then there is no tail-loss
1207 * probe thus we have to let rxtshift go up to 3.
1208 */
1209 tcp_heuristic_tfo_middlebox(tp);
1210
1211 so->so_error = ENODATA;
1212 soevent(so,
1213 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1214 sorwakeup(so);
1215 sowwakeup(so);
1216
1217 tp->t_tfo_stats |= TFO_S_SEND_BLACKHOLE;
1218 tcpstat.tcps_tfo_sndblackhole++;
1219 }
1220
1221 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1222 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1223 (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&
1224 tp->t_rxtshift > 3) {
1225 if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
1226 tcp_heuristic_tfo_middlebox(tp);
1227
1228 so->so_error = ENODATA;
1229 soevent(so,
1230 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1231 sorwakeup(so);
1232 sowwakeup(so);
1233 }
1234 }
1235
1236 if (tp->t_state == TCPS_SYN_SENT) {
1237 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_SYNRMXT) != 0 &&
1238 if_link_heuristics_enabled(outifp)) {
1239 IF_TCP_STATINC(outifp, linkheur_synrxmt);
1240 /*
1241 * The following increases the RTO by the expected backoff.
1242 *
1243 * We don't want to use TCP_REXMTVAL() as that would take
1244 * the SRTT into account. But, we are in SYN_SENT state and
1245 * thus don't have an SRTT.
1246 */
1247 rexmt = tp->t_rxtcur << ((tcp_backoff[tp->t_rxtshift] - tcp_backoff[tp->t_rxtshift - 1]) / tcp_backoff[tp->t_rxtshift - 1]);
1248 } else {
1249 rexmt = tp->t_rxtcur << ((tcp_syn_backoff[tp->t_rxtshift] - tcp_syn_backoff[tp->t_rxtshift - 1]) / tcp_syn_backoff[tp->t_rxtshift - 1]);
1250 }
1251 tp->t_stat.synrxtshift = tp->t_rxtshift;
1252 tp->t_stat.rxmitsyns++;
1253
1254 /* When retransmitting, disable TFO */
1255 if (TFO_ENABLED(tp) &&
1256 !(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE)) {
1257 tcp_disable_tfo(tp);
1258 tp->t_tfo_flags |= TFO_F_SYN_LOSS;
1259 }
1260 } else {
1261 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1262 }
1263 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX,
1264 TCP_ADD_REXMTSLOP(tp));
1265
1266 tcp_set_rto(tp);
1267
1268 TCP_LOG_RTT_INFO(tp);
1269
1270 if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb)) {
1271 goto fc_output;
1272 }
1273
1274 tcp_free_sackholes(tp);
1275 if (TCP_RACK_ENABLED(tp)) {
1276 tcp_segs_clear_sacked(tp);
1277 tcp_rack_loss_on_rto(tp, true);
1278 }
1279 /*
1280 * Check for potential Path MTU Discovery Black Hole
1281 */
1282 if (tcp_pmtud_black_hole_detect &&
1283 !(tp->t_flagsext & TF_NOBLACKHOLE_DETECTION) &&
1284 (tp->t_state == TCPS_ESTABLISHED)) {
1285 if ((tp->t_flags & TF_PMTUD) &&
1286 tp->t_pmtud_lastseg_size > tcp_pmtud_black_holed_next_mss(tp) &&
1287 tp->t_rxtshift == 2) {
1288 /*
1289 * Enter Path MTU Black-hole Detection mechanism:
1290 * - Disable Path MTU Discovery (IP "DF" bit).
1291 * - Reduce MTU to lower value than what we
1292 * negotiated with the peer.
1293 */
1294 /* Disable Path MTU Discovery for now */
1295 tp->t_flags &= ~TF_PMTUD;
1296 /* Record that we may have found a black hole */
1297 tp->t_flags |= TF_BLACKHOLE;
1298 optlen = tp->t_maxopd - tp->t_maxseg;
1299 /* Keep track of previous MSS */
1300 tp->t_pmtud_saved_maxopd = tp->t_maxopd;
1301 tp->t_pmtud_start_ts = tcp_now;
1302 if (tp->t_pmtud_start_ts == 0) {
1303 tp->t_pmtud_start_ts++;
1304 }
1305 /* Reduce the MSS to intermediary value */
1306 tp->t_maxopd = tcp_pmtud_black_holed_next_mss(tp);
1307 tp->t_maxseg = tp->t_maxopd - optlen;
1308
1309 /*
1310 * Reset the slow-start flight size
1311 * as it may depend on the new MSS
1312 */
1313 if (CC_ALGO(tp)->cwnd_init != NULL) {
1314 CC_ALGO(tp)->cwnd_init(tp);
1315 }
1316 tp->snd_cwnd = tp->t_maxseg;
1317
1318 if (TCP_USE_RLEDBAT(tp, so) &&
1319 tcp_cc_rledbat.rwnd_init != NULL) {
1320 tcp_cc_rledbat.rwnd_init(tp);
1321 }
1322 }
1323 /*
1324 * If further retransmissions are still
1325 * unsuccessful with a lowered MTU, maybe this
1326 * isn't a Black Hole and we restore the previous
1327 * MSS and blackhole detection flags.
1328 */
1329 else {
1330 if ((tp->t_flags & TF_BLACKHOLE) &&
1331 (tp->t_rxtshift > 4)) {
1332 tcp_pmtud_revert_segment_size(tp);
1333 tp->snd_cwnd = tp->t_maxseg;
1334 }
1335 }
1336 }
1337
1338 /*
1339 * Disable rfc1323 and rfc1644 if we haven't got any
1340 * response to our SYN (after we reach the threshold)
1341 * to work-around some broken terminal servers (most of
1342 * which have hopefully been retired) that have bad VJ
1343 * header compression code which trashes TCP segments
1344 * containing unknown-to-them TCP options.
1345 * Do this only on non-local connections.
1346 */
1347 if (tp->t_state == TCPS_SYN_SENT &&
1348 tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) {
1349 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP);
1350 }
1351
1352 /*
1353 * If losing, let the lower level know and try for
1354 * a better route. Also, if we backed off this far,
1355 * our srtt estimate is probably bogus. Clobber it
1356 * so we'll take the next rtt measurement as our srtt;
1357 * move the current srtt into rttvar to keep the current
1358 * retransmit times until then.
1359 */
1360 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
1361 if (!(tp->t_inpcb->inp_vflag & INP_IPV4)) {
1362 in6_losing(tp->t_inpcb);
1363 } else {
1364 in_losing(tp->t_inpcb);
1365 }
1366 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
1367 tp->t_srtt = 0;
1368 }
1369 tp->snd_nxt = tp->snd_una;
1370 /*
1371 * Note: We overload snd_recover to function also as the
1372 * snd_last variable described in RFC 2582
1373 */
1374 tp->snd_recover = tp->snd_max;
1375 /*
1376 * Force a segment to be sent.
1377 */
1378 tp->t_flags |= TF_ACKNOW;
1379
1380 /*
1381 * If timing a segment in this window, stop the timer
1382 * except when we are in connecting states on cellular
1383 * interfaces
1384 */
1385 if (tp->t_state >= TCPS_ESTABLISHED || (outifp != NULL &&
1386 IFNET_IS_CELLULAR(outifp) == false)) {
1387 tp->t_rtttime = 0;
1388 }
1389
1390 if (!IN_FASTRECOVERY(tp) && tp->t_rxtshift == 1) {
1391 tcpstat.tcps_tailloss_rto++;
1392 }
1393
1394 /*
1395 * RFC 5681 says: when a TCP sender detects segment loss
1396 * using retransmit timer and the given segment has already
1397 * been retransmitted by way of the retransmission timer at
1398 * least once, the value of ssthresh is held constant
1399 */
1400 if (tp->t_rxtshift == 1 &&
1401 CC_ALGO(tp)->after_timeout != NULL) {
1402 CC_ALGO(tp)->after_timeout(tp);
1403 /*
1404 * CWR notifications are to be sent on new data
1405 * right after Fast Retransmits and ECE
1406 * notification receipts.
1407 */
1408 if (!tp->accurate_ecn_on && TCP_ECN_ENABLED(tp)) {
1409 tp->ecn_flags |= TE_SENDCWR;
1410 }
1411 }
1412
1413 EXIT_FASTRECOVERY(tp);
1414
1415 /* Exit cwnd non validated phase */
1416 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
1417
1418
1419 fc_output:
1420 tcp_ccdbg_trace(tp, NULL, TCP_CC_REXMT_TIMEOUT);
1421
1422 (void) tcp_output(tp);
1423 break;
1424
1425 /*
1426 * Persistance timer into zero window.
1427 * Force a byte to be output, if possible.
1428 */
1429 case TCPT_PERSIST:
1430 tcpstat.tcps_persisttimeo++;
1431 /*
1432 * Hack: if the peer is dead/unreachable, we do not
1433 * time out if the window is closed. After a full
1434 * backoff, drop the connection if the idle time
1435 * (no responses to probes) reaches the maximum
1436 * backoff that we would use if retransmitting.
1437 *
1438 * Drop the connection if we reached the maximum allowed time for
1439 * Zero Window Probes without a non-zero update from the peer.
1440 * See rdar://5805356
1441 */
1442 if ((tp->t_rxtshift == TCP_MAXRXTSHIFT &&
1443 (idle_time >= tcp_maxpersistidle ||
1444 idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) ||
1445 ((tp->t_persist_stop != 0) &&
1446 TSTMP_LEQ(tp->t_persist_stop, tcp_now))) {
1447 TCP_LOG_DROP_PCB(NULL, NULL, tp, false, "persist timeout drop");
1448 tcpstat.tcps_persistdrop++;
1449 soevent(so,
1450 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1451 tp = tcp_drop(tp, ETIMEDOUT);
1452 break;
1453 }
1454 tcp_setpersist(tp);
1455 tp->t_flagsext |= TF_FORCE;
1456 (void) tcp_output(tp);
1457 tp->t_flagsext &= ~TF_FORCE;
1458 break;
1459
1460 /*
1461 * Keep-alive timer went off; send something
1462 * or drop connection if idle for too long.
1463 */
1464 case TCPT_KEEP:
1465 #if FLOW_DIVERT
1466 if (tp->t_inpcb->inp_socket->so_flags & SOF_FLOW_DIVERT) {
1467 break;
1468 }
1469 #endif /* FLOW_DIVERT */
1470
1471 tcpstat.tcps_keeptimeo++;
1472 #if MPTCP
1473 /*
1474 * Regular TCP connections do not send keepalives after closing
1475 * MPTCP must not also, after sending Data FINs.
1476 */
1477 struct mptcb *mp_tp = tptomptp(tp);
1478 if ((tp->t_mpflags & TMPF_MPTCP_TRUE) &&
1479 (tp->t_state > TCPS_ESTABLISHED)) {
1480 goto dropit;
1481 } else if (mp_tp != NULL) {
1482 if ((mptcp_ok_to_keepalive(mp_tp) == 0)) {
1483 goto dropit;
1484 }
1485 }
1486 #endif /* MPTCP */
1487 if (tp->t_state < TCPS_ESTABLISHED) {
1488 goto dropit;
1489 }
1490 if ((always_keepalive ||
1491 (tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) ||
1492 (tp->t_flagsext & TF_DETECT_READSTALL) ||
1493 (tp->t_tfo_probe_state == TFO_PROBE_PROBING)) &&
1494 (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
1495 if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp)) {
1496 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1497 "keep alive timeout drop");
1498 goto dropit;
1499 }
1500
1501 if (tcp_send_keep_alive(tp)) {
1502 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1503 tp->t_rtimo_probes++;
1504 }
1505
1506 TCP_LOG_KEEP_ALIVE(tp, idle_time);
1507 }
1508
1509 tp->t_timer[TCPT_KEEP] = tcp_offset_from_start(tp,
1510 TCP_CONN_KEEPINTVL(tp));
1511 } else {
1512 tp->t_timer[TCPT_KEEP] = tcp_offset_from_start(tp,
1513 TCP_CONN_KEEPIDLE(tp));
1514 }
1515 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1516 bool reenable_probe = false;
1517 /*
1518 * The keep alive packets sent to detect a read
1519 * stall did not get a response from the
1520 * peer. Generate more keep-alives to confirm this.
1521 * If the number of probes sent reaches the limit,
1522 * generate an event.
1523 */
1524 if (tp->t_adaptive_rtimo > 0) {
1525 if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) {
1526 /* Generate an event */
1527 soevent(so,
1528 (SO_FILT_HINT_LOCKED |
1529 SO_FILT_HINT_ADAPTIVE_RTIMO));
1530 tcp_keepalive_reset(tp);
1531 } else {
1532 reenable_probe = true;
1533 }
1534 } else if (outifp != NULL &&
1535 (outifp->if_eflags & IFEF_PROBE_CONNECTIVITY) &&
1536 tp->t_rtimo_probes <= TCP_CONNECTIVITY_PROBES_MAX) {
1537 reenable_probe = true;
1538 } else {
1539 tp->t_flagsext &= ~TF_DETECT_READSTALL;
1540 }
1541 if (reenable_probe) {
1542 int ind = min(tp->t_rtimo_probes,
1543 TCP_MAXRXTSHIFT);
1544 tp->t_timer[TCPT_KEEP] = tcp_offset_from_start(
1545 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp));
1546 }
1547 }
1548 if (tp->t_tfo_probe_state == TFO_PROBE_PROBING) {
1549 int ind;
1550
1551 tp->t_tfo_probes++;
1552 ind = min(tp->t_tfo_probes, TCP_MAXRXTSHIFT);
1553
1554 /*
1555 * We take the minimum among the time set by true
1556 * keepalive (see above) and the backoff'd RTO. That
1557 * way we backoff in case of packet-loss but will never
1558 * timeout slower than regular keepalive due to the
1559 * backing off.
1560 */
1561 tp->t_timer[TCPT_KEEP] = min(tcp_offset_from_start(
1562 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)),
1563 tp->t_timer[TCPT_KEEP]);
1564 } else if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1565 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1566 tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) {
1567 /* Still no data! Let's assume a TFO-error and err out... */
1568 tcp_heuristic_tfo_middlebox(tp);
1569
1570 so->so_error = ENODATA;
1571 soevent(so,
1572 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1573 sorwakeup(so);
1574 tp->t_tfo_stats |= TFO_S_RECV_BLACKHOLE;
1575 tcpstat.tcps_tfo_blackhole++;
1576 }
1577 break;
1578 case TCPT_DELACK:
1579 if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) {
1580 tp->t_flags &= ~TF_DELACK;
1581 tp->t_timer[TCPT_DELACK] = 0;
1582 tp->t_flags |= TF_ACKNOW;
1583
1584 tp->t_forced_acks = TCP_FORCED_ACKS_COUNT;
1585
1586 /*
1587 * If we are measuring inter packet arrival jitter
1588 * for throttling a connection, this delayed ack
1589 * might be the reason for accumulating some
1590 * jitter. So let's restart the measurement.
1591 */
1592 CLEAR_IAJ_STATE(tp);
1593
1594 tcpstat.tcps_delack++;
1595 tp->t_stat.delayed_acks_sent++;
1596 (void) tcp_output(tp);
1597 }
1598 break;
1599
1600 #if MPTCP
1601 case TCPT_JACK_RXMT:
1602 if ((tp->t_state == TCPS_ESTABLISHED) &&
1603 (tp->t_mpflags & TMPF_PREESTABLISHED) &&
1604 (tp->t_mpflags & TMPF_JOINED_FLOW)) {
1605 if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) {
1606 tcpstat.tcps_timeoutdrop++;
1607 soevent(so,
1608 (SO_FILT_HINT_LOCKED |
1609 SO_FILT_HINT_TIMEOUT));
1610 tp = tcp_drop(tp, tp->t_softerror ?
1611 tp->t_softerror : ETIMEDOUT);
1612 break;
1613 }
1614 tcpstat.tcps_join_rxmts++;
1615 tp->t_mpflags |= TMPF_SND_JACK;
1616 tp->t_flags |= TF_ACKNOW;
1617
1618 /*
1619 * No backoff is implemented for simplicity for this
1620 * corner case.
1621 */
1622 (void) tcp_output(tp);
1623 }
1624 break;
1625 case TCPT_CELLICON:
1626 {
1627 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1628
1629 tp->t_timer[TCPT_CELLICON] = 0;
1630
1631 if (mpte->mpte_cellicon_increments == 0) {
1632 /* Cell-icon not set by this connection */
1633 break;
1634 }
1635
1636 if (TSTMP_LT(mpte->mpte_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, tcp_now)) {
1637 mptcp_unset_cellicon(mpte, NULL, 1);
1638 }
1639
1640 if (mpte->mpte_cellicon_increments) {
1641 tp->t_timer[TCPT_CELLICON] = tcp_offset_from_start(tp, MPTCP_CELLICON_TOGGLE_RATE);
1642 }
1643
1644 break;
1645 }
1646 #endif /* MPTCP */
1647
1648 case TCPT_PTO:
1649 {
1650 int32_t ret = 0;
1651
1652 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1653 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1654 }
1655 /*
1656 * Check if the connection is in the right state to
1657 * send a probe
1658 */
1659 if ((tp->t_state != TCPS_ESTABLISHED ||
1660 tp->t_rxtshift > 0 ||
1661 tp->snd_max == tp->snd_una ||
1662 !SACK_ENABLED(tp) || IN_FASTRECOVERY(tp)) &&
1663 !(tp->t_flagsext & TF_IF_PROBING)) {
1664 break;
1665 }
1666
1667 /*
1668 * When the interface state is changed explicitly reset the retransmission
1669 * timer state for both SYN and data packets because we do not want to
1670 * wait unnecessarily or timeout too quickly if the link characteristics
1671 * have changed drastically
1672 */
1673 if (tp->t_flagsext & TF_IF_PROBING) {
1674 tp->t_rxtshift = 0;
1675 if (tp->t_state == TCPS_SYN_SENT) {
1676 tp->t_stat.synrxtshift = tp->t_rxtshift;
1677 }
1678 /*
1679 * Reset to the the default RTO
1680 */
1681 tp->t_srtt = TCPTV_SRTTBASE;
1682 tp->t_rttvar =
1683 ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
1684 tp->t_rttmin = TCPTV_REXMTMIN;
1685 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
1686 tp->t_rttmin, TCPTV_REXMTMAX, TCP_ADD_REXMTSLOP(tp));
1687 TCP_LOG_RTT_INFO(tp);
1688 }
1689
1690 if (tp->t_state == TCPS_SYN_SENT) {
1691 /*
1692 * The PTO for SYN_SENT reinitializes TCP as if it was a fresh
1693 * connection attempt
1694 */
1695 tp->snd_nxt = tp->snd_una;
1696 /*
1697 * Note: We overload snd_recover to function also as the
1698 * snd_last variable described in RFC 2582
1699 */
1700 tp->snd_recover = tp->snd_max;
1701 /*
1702 * Force a segment to be sent.
1703 */
1704 tp->t_flags |= TF_ACKNOW;
1705
1706 /* If timing a segment in this window, stop the timer */
1707 tp->t_rtttime = 0;
1708
1709 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1710 } else {
1711 int32_t snd_len;
1712
1713 /*
1714 * If there is no new data to send or if the
1715 * connection is limited by receive window then
1716 * retransmit the last segment, otherwise send
1717 * new data.
1718 */
1719 snd_len = min(so->so_snd.sb_cc, tp->snd_wnd)
1720 - (tp->snd_max - tp->snd_una);
1721 if (snd_len > 0) {
1722 tp->snd_nxt = tp->snd_max;
1723 tp->t_flagsext &= ~TF_TLP_IS_RETRANS;
1724 } else {
1725 snd_len = min((tp->snd_max - tp->snd_una),
1726 tp->t_maxseg);
1727 tp->snd_nxt = tp->snd_max - snd_len;
1728 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1729 }
1730 }
1731
1732 tcpstat.tcps_pto++;
1733 if (tp->t_flagsext & TF_IF_PROBING) {
1734 tcpstat.tcps_probe_if++;
1735 }
1736
1737 /* If timing a segment in this window, stop the timer */
1738 tp->t_rtttime = 0;
1739 /* Note that tail loss probe is being sent. Exclude IF probe */
1740 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1741 tp->t_flagsext |= TF_SENT_TLPROBE;
1742 tp->t_tlpstart = tcp_now;
1743 }
1744
1745 tp->snd_cwnd += tp->t_maxseg;
1746 /*
1747 * When tail-loss-probe fires, we reset the RTO timer, because
1748 * a probe just got sent, so we are good to push out the timer.
1749 *
1750 * Set to 0 to ensure that tcp_output() will reschedule it
1751 */
1752 tp->t_timer[TCPT_REXMT] = 0;
1753 ret = tcp_output(tp);
1754
1755 #if (DEBUG || DEVELOPMENT)
1756 if ((tp->t_flagsext & TF_IF_PROBING) &&
1757 ((IFNET_IS_COMPANION_LINK(tp->t_inpcb->inp_last_outifp)) ||
1758 tp->t_state == TCPS_SYN_SENT)) {
1759 if (ret == 0 && tcp_probe_if_fix_port > 0 &&
1760 tcp_probe_if_fix_port <= IPPORT_HILASTAUTO) {
1761 tp->t_timer[TCPT_REXMT] = 0;
1762 tcp_set_lotimer_index(tp);
1763 }
1764
1765 os_log(OS_LOG_DEFAULT,
1766 "%s: sent %s probe for %u > %u on interface %s"
1767 " (%u) %s(%d)",
1768 __func__,
1769 tp->t_state == TCPS_SYN_SENT ? "SYN" : "data",
1770 ntohs(tp->t_inpcb->inp_lport),
1771 ntohs(tp->t_inpcb->inp_fport),
1772 if_name(tp->t_inpcb->inp_last_outifp),
1773 tp->t_inpcb->inp_last_outifp->if_index,
1774 ret == 0 ? "succeeded" :"failed", ret);
1775 }
1776 #endif /* DEBUG || DEVELOPMENT */
1777
1778 /*
1779 * When there is data (or a SYN) to send, the above call to
1780 * tcp_output() should have armed either the REXMT or the
1781 * PERSIST timer. If it didn't, something is wrong and this
1782 * connection would idle around forever. Let's make sure that
1783 * at least the REXMT timer is set.
1784 */
1785 if (tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0 &&
1786 (tp->t_inpcb->inp_socket->so_snd.sb_cc != 0 || tp->t_state == TCPS_SYN_SENT ||
1787 tp->t_state == TCPS_SYN_RECEIVED)) {
1788 tcp_set_rto(tp);
1789
1790 tcp_check_timer_state(tp);
1791 }
1792 tp->snd_cwnd -= tp->t_maxseg;
1793
1794 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1795 tp->t_tlphighrxt = tp->snd_nxt;
1796 tp->t_tlphightrxt_persist = tp->snd_nxt;
1797 }
1798 break;
1799 }
1800 case TCPT_DELAYFR:
1801 tp->t_flagsext &= ~TF_DELAY_RECOVERY;
1802
1803 /*
1804 * Don't do anything if one of the following is true:
1805 * - the connection is already in recovery
1806 * - sequence until snd_recover has been acknowledged.
1807 * - retransmit timeout has fired
1808 */
1809 if (IN_FASTRECOVERY(tp) ||
1810 SEQ_GEQ(tp->snd_una, tp->snd_recover) ||
1811 tp->t_rxtshift > 0) {
1812 break;
1813 }
1814
1815 VERIFY(SACK_ENABLED(tp));
1816 tcp_rexmt_save_state(tp);
1817 if (CC_ALGO(tp)->pre_fr != NULL) {
1818 CC_ALGO(tp)->pre_fr(tp);
1819 if (!tp->accurate_ecn_on && TCP_ECN_ENABLED(tp)) {
1820 tp->ecn_flags |= TE_SENDCWR;
1821 }
1822 }
1823 ENTER_FASTRECOVERY(tp);
1824
1825 tp->t_timer[TCPT_REXMT] = 0;
1826 tcpstat.tcps_sack_recovery_episode++;
1827 tp->t_sack_recovery_episode++;
1828 tp->snd_cwnd = tp->t_maxseg;
1829 tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY);
1830 (void) tcp_output(tp);
1831 break;
1832
1833 dropit:
1834 tcpstat.tcps_keepdrops++;
1835 soevent(so,
1836 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1837 tp = tcp_drop(tp, ETIMEDOUT);
1838 break;
1839 case TCPT_REORDER:
1840 if (TCP_RACK_ENABLED(tp)) {
1841 tcp_rack_reordering_timeout(tp, 0);
1842 }
1843 break;
1844 }
1845 return tp;
1846 }
1847
1848 /* Remove a timer entry from timer list */
1849 void
tcp_remove_timer(struct tcpcb * tp)1850 tcp_remove_timer(struct tcpcb *tp)
1851 {
1852 struct tcptimerlist *listp = &tcp_timer_list;
1853
1854 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
1855 if (!(TIMER_IS_ON_LIST(tp))) {
1856 return;
1857 }
1858 lck_mtx_lock(&listp->mtx);
1859
1860 if (listp->next_te != NULL && listp->next_te == &tp->tentry) {
1861 listp->next_te = LIST_NEXT(&tp->tentry, te_le);
1862 }
1863
1864 LIST_REMOVE(&tp->tentry, te_le);
1865 /*
1866 * The use count has been incremented when the PCB
1867 * was placed on the timer list, and needs to be decremented.
1868 * As a safety precaution, we are checking against underflow.
1869 */
1870 if (__improbable(tp->t_inpcb->inp_socket->so_usecount == 0)) {
1871 TCP_LOG(tp, "%s: inpcb socket so_usecount underflow "
1872 " when removing timer entry\n", __func__);
1873 } else {
1874 tp->t_inpcb->inp_socket->so_usecount--;
1875 }
1876
1877 tp->t_flags &= ~(TF_TIMER_ONLIST);
1878
1879 listp->entries--;
1880
1881 tp->tentry.te_le.le_next = NULL;
1882 tp->tentry.te_le.le_prev = NULL;
1883
1884 lck_mtx_unlock(&listp->mtx);
1885 }
1886
1887 /*
1888 * Function to check if the timerlist needs to be rescheduled to run
1889 * the timer entry correctly. Basically, this is to check if we can avoid
1890 * taking the list lock.
1891 */
1892
1893 static boolean_t
need_to_resched_timerlist(u_int32_t runtime,u_int16_t mode)1894 need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode)
1895 {
1896 struct tcptimerlist *listp = &tcp_timer_list;
1897 int32_t diff;
1898
1899 /*
1900 * If the list is being processed then the state of the list is
1901 * in flux. In this case always acquire the lock and set the state
1902 * correctly.
1903 */
1904 if (listp->running) {
1905 return TRUE;
1906 }
1907
1908 if (!listp->scheduled) {
1909 return TRUE;
1910 }
1911
1912 diff = timer_diff(listp->runtime, 0, runtime, 0);
1913 if (diff <= 0) {
1914 /* The list is going to run before this timer */
1915 return FALSE;
1916 } else {
1917 if (mode & TCP_TIMERLIST_10MS_MODE) {
1918 if (diff <= TCP_TIMER_10MS_QUANTUM) {
1919 return FALSE;
1920 }
1921 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
1922 if (diff <= TCP_TIMER_100MS_QUANTUM) {
1923 return FALSE;
1924 }
1925 } else {
1926 if (diff <= TCP_TIMER_500MS_QUANTUM) {
1927 return FALSE;
1928 }
1929 }
1930 }
1931 return TRUE;
1932 }
1933
1934 void
tcp_sched_timerlist(uint32_t offset)1935 tcp_sched_timerlist(uint32_t offset)
1936 {
1937 uint64_t deadline = 0;
1938 struct tcptimerlist *listp = &tcp_timer_list;
1939
1940 LCK_MTX_ASSERT(&listp->mtx, LCK_MTX_ASSERT_OWNED);
1941
1942 offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);
1943 listp->runtime = tcp_now + offset;
1944 listp->schedtime = tcp_now;
1945 if (listp->runtime == 0) {
1946 listp->runtime++;
1947 offset++;
1948 }
1949
1950 clock_interval_to_deadline(offset, USEC_PER_SEC, &deadline);
1951
1952 thread_call_enter_delayed(listp->call, deadline);
1953 listp->scheduled = TRUE;
1954 }
1955
1956 /*
1957 * Function to run the timers for a connection.
1958 *
1959 * Returns the offset of next timer to be run for this connection which
1960 * can be used to reschedule the timerlist.
1961 *
1962 * te_mode is an out parameter that indicates the modes of active
1963 * timers for this connection.
1964 */
1965 u_int32_t
tcp_run_conn_timer(struct tcpcb * tp,u_int16_t * te_mode,u_int16_t probe_if_index)1966 tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode,
1967 u_int16_t probe_if_index)
1968 {
1969 struct socket *so;
1970 u_int16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE;
1971 u_int32_t timer_val, offset = 0, lo_timer = 0;
1972 int32_t diff;
1973 boolean_t needtorun[TCPT_NTIMERS];
1974 int count = 0;
1975
1976 VERIFY(tp != NULL);
1977 bzero(needtorun, sizeof(needtorun));
1978 *te_mode = 0;
1979
1980 socket_lock(tp->t_inpcb->inp_socket, 1);
1981
1982 so = tp->t_inpcb->inp_socket;
1983 /* Release the want count on inp */
1984 if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1)
1985 == WNT_STOPUSING) {
1986 if (TIMER_IS_ON_LIST(tp)) {
1987 tcp_remove_timer(tp);
1988 }
1989
1990 /* Looks like the TCP connection got closed while we
1991 * were waiting for the lock.. Done
1992 */
1993 goto done;
1994 }
1995
1996 /*
1997 * If this connection is over an interface that needs to
1998 * be probed, send probe packets to reinitiate communication.
1999 */
2000 if (TCP_IF_STATE_CHANGED(tp, probe_if_index)) {
2001 tp->t_flagsext |= TF_IF_PROBING;
2002 tcp_timers(tp, TCPT_PTO);
2003 tp->t_timer[TCPT_PTO] = 0;
2004 tp->t_flagsext &= ~TF_IF_PROBING;
2005 }
2006
2007 /*
2008 * Since the timer thread needs to wait for tcp lock, it may race
2009 * with another thread that can cancel or reschedule the timer
2010 * that is about to run. Check if we need to run anything.
2011 */
2012 if ((index = tp->tentry.te_index) == TCPT_NONE) {
2013 goto done;
2014 }
2015
2016 timer_val = tp->t_timer[index];
2017
2018 diff = timer_diff(tp->tentry.te_runtime, 0, tcp_now, 0);
2019 if (diff > 0) {
2020 if (tp->tentry.te_index != TCPT_NONE) {
2021 offset = diff;
2022 *(te_mode) = tp->tentry.te_mode;
2023 }
2024 goto done;
2025 }
2026
2027 tp->t_timer[index] = 0;
2028 if (timer_val > 0) {
2029 tp = tcp_timers(tp, index);
2030 if (tp == NULL) {
2031 goto done;
2032 }
2033 }
2034
2035 /*
2036 * Check if there are any other timers that need to be run.
2037 * While doing it, adjust the timer values wrt tcp_now.
2038 */
2039 tp->tentry.te_mode = 0;
2040 for (i = 0; i < TCPT_NTIMERS; ++i) {
2041 if (tp->t_timer[i] != 0) {
2042 diff = timer_diff(tp->tentry.te_timer_start,
2043 tp->t_timer[i], tcp_now, 0);
2044 if (diff <= 0) {
2045 needtorun[i] = TRUE;
2046 count++;
2047 } else {
2048 tp->t_timer[i] = diff;
2049 needtorun[i] = FALSE;
2050 if (lo_timer == 0 || diff < lo_timer) {
2051 lo_timer = diff;
2052 lo_index = i;
2053 }
2054 TCP_SET_TIMER_MODE(tp->tentry.te_mode, i);
2055 }
2056 }
2057 }
2058
2059 tp->tentry.te_timer_start = tcp_now;
2060 tp->tentry.te_index = lo_index;
2061 VERIFY(tp->tentry.te_index == TCPT_NONE || tp->tentry.te_mode > 0);
2062
2063 if (tp->tentry.te_index != TCPT_NONE) {
2064 tp->tentry.te_runtime = tp->tentry.te_timer_start +
2065 tp->t_timer[tp->tentry.te_index];
2066 if (tp->tentry.te_runtime == 0) {
2067 tp->tentry.te_runtime++;
2068 }
2069 }
2070
2071 if (count > 0) {
2072 /* run any other timers outstanding at this time. */
2073 for (i = 0; i < TCPT_NTIMERS; ++i) {
2074 if (needtorun[i]) {
2075 tp->t_timer[i] = 0;
2076 tp = tcp_timers(tp, i);
2077 if (tp == NULL) {
2078 offset = 0;
2079 *(te_mode) = 0;
2080 goto done;
2081 }
2082 }
2083 }
2084 tcp_set_lotimer_index(tp);
2085 }
2086
2087 if (tp->tentry.te_index < TCPT_NONE) {
2088 offset = tp->t_timer[tp->tentry.te_index];
2089 *(te_mode) = tp->tentry.te_mode;
2090 }
2091
2092 done:
2093 if (tp != NULL && tp->tentry.te_index == TCPT_NONE) {
2094 tcp_remove_timer(tp);
2095 offset = 0;
2096 }
2097
2098 socket_unlock(so, 1);
2099 return offset;
2100 }
2101
2102 static void
tcp_timer_update_drift_stats(struct tcptimerlist * listp)2103 tcp_timer_update_drift_stats(struct tcptimerlist *listp)
2104 {
2105 int32_t drift = tcp_now - listp->runtime;
2106 if (drift <= 1) {
2107 tcpstat.tcps_timer_drift_le_1_ms++;
2108 } else if (drift <= 10) {
2109 tcpstat.tcps_timer_drift_le_10_ms++;
2110 } else if (drift <= 20) {
2111 tcpstat.tcps_timer_drift_le_20_ms++;
2112 } else if (drift <= 50) {
2113 tcpstat.tcps_timer_drift_le_50_ms++;
2114 } else if (drift <= 100) {
2115 tcpstat.tcps_timer_drift_le_100_ms++;
2116 } else if (drift <= 200) {
2117 tcpstat.tcps_timer_drift_le_200_ms++;
2118 } else if (drift <= 500) {
2119 tcpstat.tcps_timer_drift_le_500_ms++;
2120 } else if (drift <= 1000) {
2121 tcpstat.tcps_timer_drift_le_1000_ms++;
2122 } else {
2123 tcpstat.tcps_timer_drift_gt_1000_ms++;
2124 }
2125 }
2126
2127 void
tcp_run_timerlist(void * arg1,void * arg2)2128 tcp_run_timerlist(void * arg1, void * arg2)
2129 {
2130 #pragma unused(arg1, arg2)
2131 struct tcptimerentry *te, *__single next_te;
2132 struct tcptimerlist *__single listp = &tcp_timer_list;
2133 struct tcpcb *__single tp;
2134 uint32_t next_timer = 0; /* offset of the next timer on the list */
2135 u_int16_t te_mode = 0; /* modes of all active timers in a tcpcb */
2136 u_int16_t list_mode = 0; /* cumulative of modes of all tcpcbs */
2137 uint32_t num_entries;
2138
2139 calculate_tcp_clock();
2140
2141 lck_mtx_lock(&listp->mtx);
2142
2143 tcp_timer_update_drift_stats(listp);
2144
2145 listp->started_at = tcp_now;
2146
2147 num_entries = listp->entries;
2148 listp->running = TRUE;
2149 listp->processed_count = 0;
2150
2151 LIST_FOREACH_SAFE(te, &listp->lhead, te_le, next_te) {
2152 uint32_t offset = 0;
2153 uint32_t runtime = te->te_runtime;
2154
2155 tp = TIMERENTRY_TO_TP(te);
2156
2157 listp->processed_count++;
2158 if (listp->processed_count > num_entries) {
2159 os_log(OS_LOG_DEFAULT, "tcp_run_timerlist done: processed_count %u > num_entries %u current %u",
2160 listp->processed_count, num_entries, listp->entries);
2161 break;
2162 }
2163
2164 /*
2165 * An interface probe may need to happen before the previously scheduled runtime
2166 */
2167 if (te->te_index < TCPT_NONE && TSTMP_GT(runtime, tcp_now) &&
2168 !TCP_IF_STATE_CHANGED(tp, listp->probe_if_index)) {
2169 offset = timer_diff(runtime, 0, tcp_now, 0);
2170 if (next_timer == 0 || offset < next_timer) {
2171 next_timer = offset;
2172 }
2173 list_mode |= te->te_mode;
2174 continue;
2175 }
2176
2177 /*
2178 * Acquire an inp wantcnt on the inpcb so that the socket
2179 * won't get detached even if tcp_close is called
2180 */
2181 if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0)
2182 == WNT_STOPUSING) {
2183 /*
2184 * Need to take socket lock because it protects
2185 * TIMER_IS_ON_LIST
2186 */
2187 lck_mtx_unlock(&listp->mtx);
2188 socket_lock(tp->t_inpcb->inp_socket, 1);
2189 tcp_remove_timer(tp);
2190 socket_unlock(tp->t_inpcb->inp_socket, 1);
2191 lck_mtx_lock(&listp->mtx);
2192 continue;
2193 }
2194
2195 /*
2196 * Store the next timerentry pointer before releasing the
2197 * list lock. If that entry has to be removed when we
2198 * release the lock, this pointer will be updated to the
2199 * element after that.
2200 */
2201 listp->next_te = next_te;
2202
2203 VERIFY_NEXT_LINK(&tp->tentry, te_le);
2204 VERIFY_PREV_LINK(&tp->tentry, te_le);
2205
2206 lck_mtx_unlock(&listp->mtx);
2207
2208 offset = tcp_run_conn_timer(tp, &te_mode,
2209 listp->probe_if_index);
2210
2211 lck_mtx_lock(&listp->mtx);
2212
2213 next_te = listp->next_te;
2214 listp->next_te = NULL;
2215
2216 if (offset > 0 && te_mode != 0) {
2217 list_mode |= te_mode;
2218
2219 if (next_timer == 0 || offset < next_timer) {
2220 next_timer = offset;
2221 }
2222 }
2223 }
2224
2225 if (!LIST_EMPTY(&listp->lhead)) {
2226 uint32_t next_mode = 0;
2227 if ((list_mode & TCP_TIMERLIST_10MS_MODE) ||
2228 (listp->pref_mode & TCP_TIMERLIST_10MS_MODE)) {
2229 next_mode = TCP_TIMERLIST_10MS_MODE;
2230 } else if ((list_mode & TCP_TIMERLIST_100MS_MODE) ||
2231 (listp->pref_mode & TCP_TIMERLIST_100MS_MODE)) {
2232 next_mode = TCP_TIMERLIST_100MS_MODE;
2233 } else {
2234 next_mode = TCP_TIMERLIST_500MS_MODE;
2235 }
2236
2237 if (next_mode != TCP_TIMERLIST_500MS_MODE) {
2238 listp->idleruns = 0;
2239 } else {
2240 /*
2241 * the next required mode is slow mode, but if
2242 * the last one was a faster mode and we did not
2243 * have enough idle runs, repeat the last mode.
2244 *
2245 * We try to keep the timer list in fast mode for
2246 * some idle time in expectation of new data.
2247 */
2248 if (listp->mode != next_mode &&
2249 listp->idleruns < timer_fastmode_idlemax) {
2250 listp->idleruns++;
2251 next_mode = listp->mode;
2252 next_timer = TCP_TIMER_100MS_QUANTUM;
2253 } else {
2254 listp->idleruns = 0;
2255 }
2256 }
2257 listp->mode = next_mode;
2258 if (listp->pref_offset != 0) {
2259 next_timer = min(listp->pref_offset, next_timer);
2260 }
2261
2262 if (listp->mode == TCP_TIMERLIST_500MS_MODE) {
2263 next_timer = max(next_timer,
2264 TCP_TIMER_500MS_QUANTUM);
2265 }
2266
2267 tcp_sched_timerlist(next_timer);
2268 } else {
2269 /*
2270 * No need to reschedule this timer, but always run
2271 * periodically at a much higher granularity.
2272 */
2273 tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);
2274 }
2275
2276 listp->running = FALSE;
2277 listp->pref_mode = 0;
2278 listp->pref_offset = 0;
2279 listp->probe_if_index = 0;
2280 listp->started_at = 0;
2281 listp->processed_count = 0;
2282
2283 lck_mtx_unlock(&listp->mtx);
2284 }
2285
2286 /*
2287 * Function to check if the timerlist needs to be rescheduled to run this
2288 * connection's timers correctly.
2289 */
2290 void
tcp_sched_timers(struct tcpcb * tp)2291 tcp_sched_timers(struct tcpcb *tp)
2292 {
2293 struct tcptimerentry *te = &tp->tentry;
2294 u_int16_t index = te->te_index;
2295 u_int16_t mode = te->te_mode;
2296 struct tcptimerlist *listp = &tcp_timer_list;
2297 int32_t offset = 0;
2298 boolean_t list_locked = FALSE;
2299
2300 if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) {
2301 /* Just return without adding the dead pcb to the list */
2302 if (TIMER_IS_ON_LIST(tp)) {
2303 tcp_remove_timer(tp);
2304 }
2305 return;
2306 }
2307
2308 if (index == TCPT_NONE) {
2309 /* Nothing to run */
2310 tcp_remove_timer(tp);
2311 return;
2312 }
2313
2314 /*
2315 * compute the offset at which the next timer for this connection
2316 * has to run.
2317 */
2318 offset = timer_diff(te->te_runtime, 0, tcp_now, 0);
2319 if (offset <= 0) {
2320 offset = 1;
2321 tcp_timer_advanced++;
2322 }
2323
2324 if (!TIMER_IS_ON_LIST(tp)) {
2325 if (!list_locked) {
2326 lck_mtx_lock(&listp->mtx);
2327 list_locked = TRUE;
2328 }
2329
2330 if (!TIMER_IS_ON_LIST(tp)) {
2331 /*
2332 * Adding the timer entry should constitute an incresed socket use count,
2333 * otherwise the socket use count may reach zero while being referenced
2334 * via the timer entry. If this happens, the timer service routine
2335 * will run into an UAF (use after free) when attempting
2336 * to get the related protocol control block.
2337 */
2338 tp->t_inpcb->inp_socket->so_usecount++;
2339
2340 LIST_INSERT_HEAD(&listp->lhead, te, te_le);
2341 tp->t_flags |= TF_TIMER_ONLIST;
2342
2343 listp->entries++;
2344 if (listp->entries > listp->maxentries) {
2345 listp->maxentries = listp->entries;
2346 }
2347
2348 /* if the list is not scheduled, just schedule it */
2349 if (!listp->scheduled) {
2350 goto schedule;
2351 }
2352 }
2353 }
2354
2355 /*
2356 * Timer entry is currently on the list, check if the list needs
2357 * to be rescheduled.
2358 */
2359 if (need_to_resched_timerlist(te->te_runtime, mode)) {
2360 tcp_resched_timerlist++;
2361
2362 if (!list_locked) {
2363 lck_mtx_lock(&listp->mtx);
2364 list_locked = TRUE;
2365 }
2366
2367 VERIFY_NEXT_LINK(te, te_le);
2368 VERIFY_PREV_LINK(te, te_le);
2369
2370 if (listp->running) {
2371 listp->pref_mode |= mode;
2372 if (listp->pref_offset == 0 ||
2373 offset < listp->pref_offset) {
2374 listp->pref_offset = offset;
2375 }
2376 } else {
2377 /*
2378 * The list could have got rescheduled while
2379 * this thread was waiting for the lock
2380 */
2381 if (listp->scheduled) {
2382 int32_t diff;
2383 diff = timer_diff(listp->runtime, 0,
2384 tcp_now, offset);
2385 if (diff <= 0) {
2386 goto done;
2387 } else {
2388 goto schedule;
2389 }
2390 } else {
2391 goto schedule;
2392 }
2393 }
2394 }
2395 goto done;
2396
2397 schedule:
2398 /*
2399 * Since a connection with timers is getting scheduled, the timer
2400 * list moves from idle to active state and that is why idlegen is
2401 * reset
2402 */
2403 if (mode & TCP_TIMERLIST_10MS_MODE) {
2404 listp->mode = TCP_TIMERLIST_10MS_MODE;
2405 listp->idleruns = 0;
2406 offset = min(offset, TCP_TIMER_10MS_QUANTUM);
2407 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
2408 if (listp->mode > TCP_TIMERLIST_100MS_MODE) {
2409 listp->mode = TCP_TIMERLIST_100MS_MODE;
2410 }
2411 listp->idleruns = 0;
2412 offset = min(offset, TCP_TIMER_100MS_QUANTUM);
2413 }
2414 tcp_sched_timerlist(offset);
2415
2416 done:
2417 if (list_locked) {
2418 lck_mtx_unlock(&listp->mtx);
2419 }
2420
2421 return;
2422 }
2423
2424 static inline void
tcp_set_lotimer_index(struct tcpcb * tp)2425 tcp_set_lotimer_index(struct tcpcb *tp)
2426 {
2427 uint16_t i, lo_index = TCPT_NONE, mode = 0;
2428 uint32_t lo_timer = 0;
2429 for (i = 0; i < TCPT_NTIMERS; ++i) {
2430 if (tp->t_timer[i] != 0) {
2431 TCP_SET_TIMER_MODE(mode, i);
2432 if (lo_timer == 0 || tp->t_timer[i] < lo_timer) {
2433 lo_timer = tp->t_timer[i];
2434 lo_index = i;
2435 }
2436 }
2437 }
2438 tp->tentry.te_index = lo_index;
2439 tp->tentry.te_mode = mode;
2440 VERIFY(tp->tentry.te_index == TCPT_NONE || tp->tentry.te_mode > 0);
2441
2442 if (tp->tentry.te_index != TCPT_NONE) {
2443 tp->tentry.te_runtime = tp->tentry.te_timer_start
2444 + tp->t_timer[tp->tentry.te_index];
2445 if (tp->tentry.te_runtime == 0) {
2446 tp->tentry.te_runtime++;
2447 }
2448 }
2449 }
2450
2451 void
tcp_check_timer_state(struct tcpcb * tp)2452 tcp_check_timer_state(struct tcpcb *tp)
2453 {
2454 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
2455
2456 if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) {
2457 return;
2458 }
2459
2460 tcp_set_lotimer_index(tp);
2461
2462 tcp_sched_timers(tp);
2463 return;
2464 }
2465
2466 static inline void
tcp_cumulative_stat(u_int32_t cur,u_int32_t * prev,u_int32_t * dest)2467 tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest)
2468 {
2469 /* handle wrap around */
2470 int32_t diff = (int32_t) (cur - *prev);
2471 if (diff > 0) {
2472 *dest = diff;
2473 } else {
2474 *dest = 0;
2475 }
2476 *prev = cur;
2477 return;
2478 }
2479
2480 static inline void
tcp_cumulative_stat64(u_int64_t cur,u_int64_t * prev,u_int64_t * dest)2481 tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest)
2482 {
2483 /* handle wrap around */
2484 int64_t diff = (int64_t) (cur - *prev);
2485 if (diff > 0) {
2486 *dest = diff;
2487 } else {
2488 *dest = 0;
2489 }
2490 *prev = cur;
2491 return;
2492 }
2493
2494 __private_extern__ void
tcp_report_stats(void)2495 tcp_report_stats(void)
2496 {
2497 struct nstat_sysinfo_data data;
2498 struct sockaddr_in dst;
2499 struct sockaddr_in6 dst6;
2500 struct rtentry *rt = NULL;
2501 static struct tcp_last_report_stats prev;
2502 u_int64_t var, uptime;
2503
2504 #define stat data.u.tcp_stats
2505 if (((uptime = net_uptime()) - tcp_last_report_time) <
2506 tcp_report_stats_interval) {
2507 return;
2508 }
2509
2510 tcp_last_report_time = uptime;
2511
2512 bzero(&data, sizeof(data));
2513 data.flags = NSTAT_SYSINFO_TCP_STATS;
2514
2515 SOCKADDR_ZERO(&dst, sizeof(dst));
2516 dst.sin_len = sizeof(dst);
2517 dst.sin_family = AF_INET;
2518
2519 /* ipv4 avg rtt */
2520 lck_mtx_lock(rnh_lock);
2521 rt = rt_lookup(TRUE, SA(&dst), NULL,
2522 rt_tables[AF_INET], IFSCOPE_NONE);
2523 lck_mtx_unlock(rnh_lock);
2524 if (rt != NULL) {
2525 RT_LOCK(rt);
2526 if (rt_primary_default(rt, rt_key(rt)) &&
2527 rt->rt_stats != NULL) {
2528 stat.ipv4_avgrtt = rt->rt_stats->nstat_avg_rtt;
2529 }
2530 RT_UNLOCK(rt);
2531 rtfree(rt);
2532 rt = NULL;
2533 }
2534
2535 /* ipv6 avg rtt */
2536 SOCKADDR_ZERO(&dst6, sizeof(dst6));
2537 dst6.sin6_len = sizeof(dst6);
2538 dst6.sin6_family = AF_INET6;
2539
2540 lck_mtx_lock(rnh_lock);
2541 rt = rt_lookup(TRUE, SA(&dst6), NULL,
2542 rt_tables[AF_INET6], IFSCOPE_NONE);
2543 lck_mtx_unlock(rnh_lock);
2544 if (rt != NULL) {
2545 RT_LOCK(rt);
2546 if (rt_primary_default(rt, rt_key(rt)) &&
2547 rt->rt_stats != NULL) {
2548 stat.ipv6_avgrtt = rt->rt_stats->nstat_avg_rtt;
2549 }
2550 RT_UNLOCK(rt);
2551 rtfree(rt);
2552 rt = NULL;
2553 }
2554
2555 /* send packet loss rate, shift by 10 for precision */
2556 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_sndrexmitpack > 0) {
2557 var = tcpstat.tcps_sndrexmitpack << 10;
2558 stat.send_plr = (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2559 }
2560
2561 /* recv packet loss rate, shift by 10 for precision */
2562 if (tcpstat.tcps_rcvpack > 0 && tcpstat.tcps_recovered_pkts > 0) {
2563 var = tcpstat.tcps_recovered_pkts << 10;
2564 stat.recv_plr = (uint32_t)((var * 100) / tcpstat.tcps_rcvpack);
2565 }
2566
2567 /* RTO after tail loss, shift by 10 for precision */
2568 if (tcpstat.tcps_sndrexmitpack > 0
2569 && tcpstat.tcps_tailloss_rto > 0) {
2570 var = tcpstat.tcps_tailloss_rto << 10;
2571 stat.send_tlrto_rate =
2572 (uint32_t)((var * 100) / tcpstat.tcps_sndrexmitpack);
2573 }
2574
2575 /* packet reordering */
2576 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) {
2577 var = tcpstat.tcps_reordered_pkts << 10;
2578 stat.send_reorder_rate =
2579 (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2580 }
2581
2582 if (tcp_ecn == 1) {
2583 stat.ecn_client_enabled = 1;
2584 stat.ecn_server_enabled = 1;
2585 }
2586
2587 tcp_cumulative_stat(tcpstat.tcps_connattempt,
2588 &prev.tcps_connattempt, &stat.connection_attempts);
2589 tcp_cumulative_stat(tcpstat.tcps_accepts,
2590 &prev.tcps_accepts, &stat.connection_accepts);
2591 tcp_cumulative_stat(tcpstat.tcps_ecn_client_setup,
2592 &prev.tcps_ecn_client_setup, &stat.ecn_client_setup);
2593 tcp_cumulative_stat(tcpstat.tcps_ecn_server_setup,
2594 &prev.tcps_ecn_server_setup, &stat.ecn_server_setup);
2595 tcp_cumulative_stat(tcpstat.tcps_ecn_client_success,
2596 &prev.tcps_ecn_client_success, &stat.ecn_client_success);
2597 tcp_cumulative_stat(tcpstat.tcps_ecn_server_success,
2598 &prev.tcps_ecn_server_success, &stat.ecn_server_success);
2599 tcp_cumulative_stat(tcpstat.tcps_ecn_not_supported,
2600 &prev.tcps_ecn_not_supported, &stat.ecn_not_supported);
2601 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_syn,
2602 &prev.tcps_ecn_lost_syn, &stat.ecn_lost_syn);
2603 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_synack,
2604 &prev.tcps_ecn_lost_synack, &stat.ecn_lost_synack);
2605 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ce,
2606 &prev.tcps_ecn_recv_ce, &stat.ecn_recv_ce);
2607 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2608 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2609 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2610 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2611 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2612 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2613 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2614 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2615 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ce,
2616 &prev.tcps_ecn_conn_recv_ce, &stat.ecn_conn_recv_ce);
2617 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ece,
2618 &prev.tcps_ecn_conn_recv_ece, &stat.ecn_conn_recv_ece);
2619 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_plnoce,
2620 &prev.tcps_ecn_conn_plnoce, &stat.ecn_conn_plnoce);
2621 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_pl_ce,
2622 &prev.tcps_ecn_conn_pl_ce, &stat.ecn_conn_pl_ce);
2623 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_nopl_ce,
2624 &prev.tcps_ecn_conn_nopl_ce, &stat.ecn_conn_nopl_ce);
2625 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_synloss,
2626 &prev.tcps_ecn_fallback_synloss, &stat.ecn_fallback_synloss);
2627 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_reorder,
2628 &prev.tcps_ecn_fallback_reorder, &stat.ecn_fallback_reorder);
2629 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_ce,
2630 &prev.tcps_ecn_fallback_ce, &stat.ecn_fallback_ce);
2631 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_rcv,
2632 &prev.tcps_tfo_syn_data_rcv, &stat.tfo_syn_data_rcv);
2633 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req_rcv,
2634 &prev.tcps_tfo_cookie_req_rcv, &stat.tfo_cookie_req_rcv);
2635 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_sent,
2636 &prev.tcps_tfo_cookie_sent, &stat.tfo_cookie_sent);
2637 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_invalid,
2638 &prev.tcps_tfo_cookie_invalid, &stat.tfo_cookie_invalid);
2639 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req,
2640 &prev.tcps_tfo_cookie_req, &stat.tfo_cookie_req);
2641 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_rcv,
2642 &prev.tcps_tfo_cookie_rcv, &stat.tfo_cookie_rcv);
2643 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_sent,
2644 &prev.tcps_tfo_syn_data_sent, &stat.tfo_syn_data_sent);
2645 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_acked,
2646 &prev.tcps_tfo_syn_data_acked, &stat.tfo_syn_data_acked);
2647 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_loss,
2648 &prev.tcps_tfo_syn_loss, &stat.tfo_syn_loss);
2649 tcp_cumulative_stat(tcpstat.tcps_tfo_blackhole,
2650 &prev.tcps_tfo_blackhole, &stat.tfo_blackhole);
2651 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_wrong,
2652 &prev.tcps_tfo_cookie_wrong, &stat.tfo_cookie_wrong);
2653 tcp_cumulative_stat(tcpstat.tcps_tfo_no_cookie_rcv,
2654 &prev.tcps_tfo_no_cookie_rcv, &stat.tfo_no_cookie_rcv);
2655 tcp_cumulative_stat(tcpstat.tcps_tfo_heuristics_disable,
2656 &prev.tcps_tfo_heuristics_disable, &stat.tfo_heuristics_disable);
2657 tcp_cumulative_stat(tcpstat.tcps_tfo_sndblackhole,
2658 &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole);
2659
2660
2661 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt,
2662 &prev.tcps_mptcp_handover_attempt, &stat.mptcp_handover_attempt);
2663 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt,
2664 &prev.tcps_mptcp_interactive_attempt, &stat.mptcp_interactive_attempt);
2665 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt,
2666 &prev.tcps_mptcp_aggregate_attempt, &stat.mptcp_aggregate_attempt);
2667 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt,
2668 &prev.tcps_mptcp_fp_handover_attempt, &stat.mptcp_fp_handover_attempt);
2669 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt,
2670 &prev.tcps_mptcp_fp_interactive_attempt, &stat.mptcp_fp_interactive_attempt);
2671 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt,
2672 &prev.tcps_mptcp_fp_aggregate_attempt, &stat.mptcp_fp_aggregate_attempt);
2673 tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback,
2674 &prev.tcps_mptcp_heuristic_fallback, &stat.mptcp_heuristic_fallback);
2675 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback,
2676 &prev.tcps_mptcp_fp_heuristic_fallback, &stat.mptcp_fp_heuristic_fallback);
2677 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi,
2678 &prev.tcps_mptcp_handover_success_wifi, &stat.mptcp_handover_success_wifi);
2679 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell,
2680 &prev.tcps_mptcp_handover_success_cell, &stat.mptcp_handover_success_cell);
2681 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success,
2682 &prev.tcps_mptcp_interactive_success, &stat.mptcp_interactive_success);
2683 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success,
2684 &prev.tcps_mptcp_aggregate_success, &stat.mptcp_aggregate_success);
2685 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi,
2686 &prev.tcps_mptcp_fp_handover_success_wifi, &stat.mptcp_fp_handover_success_wifi);
2687 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell,
2688 &prev.tcps_mptcp_fp_handover_success_cell, &stat.mptcp_fp_handover_success_cell);
2689 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success,
2690 &prev.tcps_mptcp_fp_interactive_success, &stat.mptcp_fp_interactive_success);
2691 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success,
2692 &prev.tcps_mptcp_fp_aggregate_success, &stat.mptcp_fp_aggregate_success);
2693 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi,
2694 &prev.tcps_mptcp_handover_cell_from_wifi, &stat.mptcp_handover_cell_from_wifi);
2695 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell,
2696 &prev.tcps_mptcp_handover_wifi_from_cell, &stat.mptcp_handover_wifi_from_cell);
2697 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi,
2698 &prev.tcps_mptcp_interactive_cell_from_wifi, &stat.mptcp_interactive_cell_from_wifi);
2699 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes,
2700 &prev.tcps_mptcp_handover_cell_bytes, &stat.mptcp_handover_cell_bytes);
2701 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes,
2702 &prev.tcps_mptcp_interactive_cell_bytes, &stat.mptcp_interactive_cell_bytes);
2703 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes,
2704 &prev.tcps_mptcp_aggregate_cell_bytes, &stat.mptcp_aggregate_cell_bytes);
2705 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes,
2706 &prev.tcps_mptcp_handover_all_bytes, &stat.mptcp_handover_all_bytes);
2707 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes,
2708 &prev.tcps_mptcp_interactive_all_bytes, &stat.mptcp_interactive_all_bytes);
2709 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes,
2710 &prev.tcps_mptcp_aggregate_all_bytes, &stat.mptcp_aggregate_all_bytes);
2711 tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi,
2712 &prev.tcps_mptcp_back_to_wifi, &stat.mptcp_back_to_wifi);
2713 tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy,
2714 &prev.tcps_mptcp_wifi_proxy, &stat.mptcp_wifi_proxy);
2715 tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy,
2716 &prev.tcps_mptcp_cell_proxy, &stat.mptcp_cell_proxy);
2717 tcp_cumulative_stat(tcpstat.tcps_mptcp_triggered_cell,
2718 &prev.tcps_mptcp_triggered_cell, &stat.mptcp_triggered_cell);
2719
2720 nstat_sysinfo_send_data(&data);
2721
2722 #undef stat
2723 }
2724
2725 void
tcp_interface_send_probe(u_int16_t probe_if_index)2726 tcp_interface_send_probe(u_int16_t probe_if_index)
2727 {
2728 int32_t offset = 0;
2729 struct tcptimerlist *listp = &tcp_timer_list;
2730
2731 /* Make sure TCP clock is up to date */
2732 calculate_tcp_clock();
2733
2734 lck_mtx_lock(&listp->mtx);
2735 if (listp->probe_if_index > 0 && listp->probe_if_index != probe_if_index) {
2736 tcpstat.tcps_probe_if_conflict++;
2737 os_log(OS_LOG_DEFAULT,
2738 "%s: probe_if_index %u conflicts with %u, tcps_probe_if_conflict %u\n",
2739 __func__, probe_if_index, listp->probe_if_index,
2740 tcpstat.tcps_probe_if_conflict);
2741 goto done;
2742 }
2743
2744 listp->probe_if_index = probe_if_index;
2745 if (listp->running) {
2746 os_log(OS_LOG_DEFAULT, "%s: timer list already running for if_index %u\n",
2747 __func__, probe_if_index);
2748 goto done;
2749 }
2750
2751 /*
2752 * Reschedule the timerlist to run within the next 10ms, which is
2753 * the fastest that we can do.
2754 */
2755 offset = TCP_TIMER_10MS_QUANTUM;
2756 if (listp->scheduled) {
2757 int32_t diff;
2758 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2759 if (diff <= 0) {
2760 /* The timer will fire sooner than what's needed */
2761 os_log(OS_LOG_DEFAULT,
2762 "%s: timer will fire sooner than needed for if_index %u\n",
2763 __func__, probe_if_index);
2764 goto done;
2765 }
2766 }
2767 listp->mode = TCP_TIMERLIST_10MS_MODE;
2768 listp->idleruns = 0;
2769
2770 tcp_sched_timerlist(offset);
2771
2772 done:
2773 lck_mtx_unlock(&listp->mtx);
2774 return;
2775 }
2776
2777 /*
2778 * Enable read probes on this connection, if:
2779 * - it is in established state
2780 * - doesn't have any data outstanding
2781 * - the outgoing ifp matches
2782 * - we have not already sent any read probes
2783 */
2784 static void
tcp_enable_read_probe(struct tcpcb * tp,struct ifnet * ifp)2785 tcp_enable_read_probe(struct tcpcb *tp, struct ifnet *ifp)
2786 {
2787 if (tp->t_state == TCPS_ESTABLISHED &&
2788 tp->snd_max == tp->snd_una &&
2789 tp->t_inpcb->inp_last_outifp == ifp &&
2790 !(tp->t_flagsext & TF_DETECT_READSTALL) &&
2791 tp->t_rtimo_probes == 0) {
2792 tp->t_flagsext |= TF_DETECT_READSTALL;
2793 tp->t_rtimo_probes = 0;
2794 tp->t_timer[TCPT_KEEP] = tcp_offset_from_start(tp,
2795 TCP_TIMER_10MS_QUANTUM);
2796 if (tp->tentry.te_index == TCPT_NONE) {
2797 tp->tentry.te_index = TCPT_KEEP;
2798 tp->tentry.te_runtime = tcp_now +
2799 TCP_TIMER_10MS_QUANTUM;
2800 } else {
2801 int32_t diff = 0;
2802
2803 /* Reset runtime to be in next 10ms */
2804 diff = timer_diff(tp->tentry.te_runtime, 0,
2805 tcp_now, TCP_TIMER_10MS_QUANTUM);
2806 if (diff > 0) {
2807 tp->tentry.te_index = TCPT_KEEP;
2808 tp->tentry.te_runtime = tcp_now +
2809 TCP_TIMER_10MS_QUANTUM;
2810 if (tp->tentry.te_runtime == 0) {
2811 tp->tentry.te_runtime++;
2812 }
2813 }
2814 }
2815 }
2816 }
2817
2818 /*
2819 * Disable read probe and reset the keep alive timer
2820 */
2821 static void
tcp_disable_read_probe(struct tcpcb * tp)2822 tcp_disable_read_probe(struct tcpcb *tp)
2823 {
2824 if (tp->t_adaptive_rtimo == 0 &&
2825 ((tp->t_flagsext & TF_DETECT_READSTALL) ||
2826 tp->t_rtimo_probes > 0)) {
2827 tcp_keepalive_reset(tp);
2828
2829 if (tp->t_mpsub) {
2830 mptcp_reset_keepalive(tp);
2831 }
2832 }
2833 }
2834
2835 /*
2836 * Reschedule the tcp timerlist in the next 10ms to re-enable read/write
2837 * probes on connections going over a particular interface.
2838 */
2839 void
tcp_probe_connectivity(struct ifnet * ifp,u_int32_t enable)2840 tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable)
2841 {
2842 int32_t offset;
2843 struct tcptimerlist *listp = &tcp_timer_list;
2844 struct inpcbinfo *pcbinfo = &tcbinfo;
2845 struct inpcb *inp, *nxt;
2846
2847 if (ifp == NULL) {
2848 return;
2849 }
2850
2851 /* update clock */
2852 calculate_tcp_clock();
2853
2854 /*
2855 * Enable keep alive timer on all connections that are
2856 * active/established on this interface.
2857 */
2858 lck_rw_lock_shared(&pcbinfo->ipi_lock);
2859
2860 LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, nxt) {
2861 struct tcpcb *tp = NULL;
2862 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) ==
2863 WNT_STOPUSING) {
2864 continue;
2865 }
2866
2867 /* Acquire lock to look at the state of the connection */
2868 socket_lock(inp->inp_socket, 1);
2869
2870 /* Release the want count */
2871 if (inp->inp_ppcb == NULL ||
2872 (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) {
2873 socket_unlock(inp->inp_socket, 1);
2874 continue;
2875 }
2876 tp = intotcpcb(inp);
2877 if (enable) {
2878 tcp_enable_read_probe(tp, ifp);
2879 } else {
2880 tcp_disable_read_probe(tp);
2881 }
2882
2883 socket_unlock(inp->inp_socket, 1);
2884 }
2885 lck_rw_done(&pcbinfo->ipi_lock);
2886
2887 lck_mtx_lock(&listp->mtx);
2888 if (listp->running) {
2889 listp->pref_mode |= TCP_TIMERLIST_10MS_MODE;
2890 goto done;
2891 }
2892
2893 /* Reschedule within the next 10ms */
2894 offset = TCP_TIMER_10MS_QUANTUM;
2895 if (listp->scheduled) {
2896 int32_t diff;
2897 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2898 if (diff <= 0) {
2899 /* The timer will fire sooner than what's needed */
2900 goto done;
2901 }
2902 }
2903 listp->mode = TCP_TIMERLIST_10MS_MODE;
2904 listp->idleruns = 0;
2905
2906 tcp_sched_timerlist(offset);
2907 done:
2908 lck_mtx_unlock(&listp->mtx);
2909 return;
2910 }
2911
2912 inline void
tcp_update_mss_core(struct tcpcb * tp,struct ifnet * ifp)2913 tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp)
2914 {
2915 struct if_cellular_status_v1 *ifsr;
2916 u_int32_t optlen;
2917 ifsr = &ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
2918 if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) {
2919 optlen = tp->t_maxopd - tp->t_maxseg;
2920
2921 if (ifsr->mss_recommended ==
2922 IF_CELL_UL_MSS_RECOMMENDED_NONE &&
2923 tp->t_cached_maxopd > 0 &&
2924 tp->t_maxopd < tp->t_cached_maxopd) {
2925 tp->t_maxopd = tp->t_cached_maxopd;
2926 tcpstat.tcps_mss_to_default++;
2927 } else if (ifsr->mss_recommended ==
2928 IF_CELL_UL_MSS_RECOMMENDED_MEDIUM &&
2929 tp->t_maxopd > tcp_mss_rec_medium) {
2930 tp->t_cached_maxopd = tp->t_maxopd;
2931 tp->t_maxopd = tcp_mss_rec_medium;
2932 tcpstat.tcps_mss_to_medium++;
2933 } else if (ifsr->mss_recommended ==
2934 IF_CELL_UL_MSS_RECOMMENDED_LOW &&
2935 tp->t_maxopd > tcp_mss_rec_low) {
2936 tp->t_cached_maxopd = tp->t_maxopd;
2937 tp->t_maxopd = tcp_mss_rec_low;
2938 tcpstat.tcps_mss_to_low++;
2939 }
2940 tp->t_maxseg = tp->t_maxopd - optlen;
2941
2942 /*
2943 * clear the cached value if it is same as the current
2944 */
2945 if (tp->t_maxopd == tp->t_cached_maxopd) {
2946 tp->t_cached_maxopd = 0;
2947 }
2948 }
2949 }
2950
2951 void
tcp_update_mss_locked(struct socket * so,struct ifnet * ifp)2952 tcp_update_mss_locked(struct socket *so, struct ifnet *ifp)
2953 {
2954 struct inpcb *inp = sotoinpcb(so);
2955 struct tcpcb *tp = intotcpcb(inp);
2956
2957 if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) {
2958 return;
2959 }
2960
2961 if (!IFNET_IS_CELLULAR(ifp)) {
2962 /*
2963 * This optimization is implemented for cellular
2964 * networks only
2965 */
2966 return;
2967 }
2968 if (tp->t_state <= TCPS_CLOSE_WAIT) {
2969 /*
2970 * If the connection is currently doing or has done PMTU
2971 * blackhole detection, do not change the MSS
2972 */
2973 if (tp->t_flags & TF_BLACKHOLE) {
2974 return;
2975 }
2976 if (ifp->if_link_status == NULL) {
2977 return;
2978 }
2979 tcp_update_mss_core(tp, ifp);
2980 }
2981 }
2982
2983 void
tcp_itimer(struct inpcbinfo * ipi)2984 tcp_itimer(struct inpcbinfo *ipi)
2985 {
2986 struct inpcb *inp, *nxt;
2987
2988 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
2989 if (tcp_itimer_done == TRUE) {
2990 tcp_itimer_done = FALSE;
2991 os_atomic_inc(&ipi->ipi_timer_req.intimer_fast, relaxed);
2992 return;
2993 }
2994 /* Upgrade failed, lost lock now take it again exclusive */
2995 lck_rw_lock_exclusive(&ipi->ipi_lock);
2996 }
2997 tcp_itimer_done = TRUE;
2998
2999 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
3000 struct socket *so;
3001 struct ifnet *ifp;
3002
3003 if (inp->inp_ppcb == NULL ||
3004 in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) {
3005 continue;
3006 }
3007 so = inp->inp_socket;
3008 ifp = inp->inp_last_outifp;
3009 socket_lock(so, 1);
3010 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
3011 socket_unlock(so, 1);
3012 continue;
3013 }
3014 so_check_extended_bk_idle_time(so);
3015 if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) {
3016 tcp_update_mss_locked(so, NULL);
3017 }
3018 socket_unlock(so, 1);
3019
3020 /*
3021 * Defunct all system-initiated background sockets if the
3022 * socket is using the cellular interface and the interface
3023 * has its LQM set to abort.
3024 */
3025 if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) &&
3026 IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) &&
3027 ifp != NULL && IFNET_IS_CELLULAR(ifp) &&
3028 (ifp->if_interface_state.valid_bitmask &
3029 IF_INTERFACE_STATE_LQM_STATE_VALID) &&
3030 ifp->if_interface_state.lqm_state ==
3031 IFNET_LQM_THRESH_ABORT) {
3032 socket_defunct(current_proc(), so,
3033 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL);
3034 }
3035 }
3036
3037 ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT);
3038 lck_rw_done(&ipi->ipi_lock);
3039 }
3040
3041 static uint32_t
tcp_offset_from_latest_tx(const struct tcpcb * tp,uint32_t offset)3042 tcp_offset_from_latest_tx(const struct tcpcb *tp, uint32_t offset)
3043 {
3044 if (TSTMP_GT(tp->t_latest_tx, tcp_now)) {
3045 return _tcp_offset_from_start(tp, offset, tp->t_latest_tx);
3046 } else {
3047 return _tcp_offset_from_start(tp, offset, tcp_now);
3048 }
3049 }
3050
3051
3052 void
tcp_set_rto(struct tcpcb * tp)3053 tcp_set_rto(struct tcpcb *tp)
3054 {
3055 struct ifnet *ifp = tp->t_inpcb->inp_last_outifp;
3056
3057 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_RTOMIN) != 0 &&
3058 ifp != NULL && if_link_heuristics_enabled(ifp)) {
3059 if (tp->t_rxtcur < tcp_link_heuristics_rto_min) {
3060 IF_TCP_STATINC(ifp, linkheur_rxmtfloor);
3061 tp->t_rxtcur = tcp_link_heuristics_rto_min;
3062 }
3063 }
3064
3065 tp->t_timer[TCPT_REXMT] = tcp_offset_from_latest_tx(tp, tp->t_rxtcur);
3066 }
3067
3068 void
tcp_set_pto(struct tcpcb * tp)3069 tcp_set_pto(struct tcpcb *tp)
3070 {
3071 uint32_t pto, srtt;
3072 struct ifnet *ifp;
3073
3074 /*
3075 * Set tail loss probe timeout if new data is being
3076 * transmitted. This will be supported only when
3077 * SACK option is enabled on a connection.
3078 *
3079 * Every time new data is sent PTO will get reset.
3080 */
3081 if (tp->t_state != TCPS_ESTABLISHED ||
3082 !SACK_ENABLED(tp) || IN_FASTRECOVERY(tp) ||
3083 tp->snd_nxt != tp->snd_max ||
3084 SEQ_LEQ(tp->snd_nxt, tp->snd_una) ||
3085 tp->t_rxtshift != 0 ||
3086 (tp->t_flagsext & (TF_SENT_TLPROBE | TF_PKTS_REORDERED)) != 0) {
3087 return;
3088 }
3089
3090 ifp = tp->t_inpcb->inp_last_outifp;
3091
3092 /*
3093 * Don't use TLP on congested link
3094 */
3095 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_NOTLP) != 0 &&
3096 if_link_heuristics_enabled(ifp)) {
3097 return;
3098 }
3099
3100 srtt = tp->t_srtt >> TCP_RTT_SHIFT;
3101 pto = 2 * srtt;
3102 if ((tp->snd_max - tp->snd_una) <= tp->t_maxseg) {
3103 pto += tcp_delack;
3104 } else {
3105 pto += 2;
3106 }
3107
3108 /* if RTO is less than PTO, choose RTO instead */
3109 if (tp->t_rxtcur < pto) {
3110 pto = tp->t_rxtcur;
3111 }
3112
3113 tp->t_timer[TCPT_PTO] = tcp_offset_from_latest_tx(tp, pto);
3114 }
3115