1 /*
2 * Copyright (c) 2000-2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_timer.c 8.2 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_timer.c,v 1.34.2.11 2001/08/22 00:59:12 silby Exp $
62 */
63
64 #include "tcp_includes.h"
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/mbuf.h>
70 #include <sys/sysctl.h>
71 #include <sys/socket.h>
72 #include <sys/socketvar.h>
73 #include <sys/protosw.h>
74 #include <sys/domain.h>
75 #include <sys/mcache.h>
76 #include <sys/queue.h>
77 #include <kern/locks.h>
78 #include <kern/cpu_number.h> /* before tcp_seq.h, for tcp_random18() */
79 #include <mach/boolean.h>
80
81 #include <net/route.h>
82 #include <net/if_var.h>
83 #include <net/ntstat.h>
84
85 #include <netinet/in.h>
86 #include <netinet/in_systm.h>
87 #include <netinet/in_pcb.h>
88 #include <netinet/in_var.h>
89 #include <netinet6/in6_pcb.h>
90 #include <netinet/ip_var.h>
91 #include <netinet/tcp.h>
92 #include <netinet/tcp_cache.h>
93 #include <netinet/tcp_fsm.h>
94 #include <netinet/tcp_seq.h>
95 #include <netinet/tcp_timer.h>
96 #include <netinet/tcp_var.h>
97 #include <netinet/tcp_cc.h>
98 #include <netinet6/tcp6_var.h>
99 #include <netinet/tcpip.h>
100 #include <netinet/tcp_log.h>
101
102 #include <sys/kdebug.h>
103 #include <mach/sdt.h>
104 #include <netinet/mptcp_var.h>
105 #include <net/content_filter.h>
106 #include <net/sockaddr_utils.h>
107
108 /* Max number of times a stretch ack can be delayed on a connection */
109 #define TCP_STRETCHACK_DELAY_THRESHOLD 5
110
111 /*
112 * If the host processor has been sleeping for too long, this is the threshold
113 * used to avoid sending stale retransmissions.
114 */
115 #define TCP_SLEEP_TOO_LONG (10 * 60 * 1000) /* 10 minutes in ms */
116
117 /* tcp timer list */
118 struct tcptimerlist tcp_timer_list;
119
120 /* List of pcbs in timewait state, protected by tcbinfo's ipi_lock */
121 struct tcptailq tcp_tw_tailq;
122
123
124 static int
125 sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
126 {
127 #pragma unused(arg2)
128 int error, temp;
129 long s, tt;
130
131 tt = *(int *)arg1;
132 s = tt * 1000 / TCP_RETRANSHZ;
133 if (tt < 0 || s > INT_MAX) {
134 return EINVAL;
135 }
136 temp = (int)s;
137
138 error = sysctl_handle_int(oidp, &temp, 0, req);
139 if (error || !req->newptr) {
140 return error;
141 }
142
143 tt = (long)temp * TCP_RETRANSHZ / 1000;
144 if (tt < 1 || tt > INT_MAX) {
145 return EINVAL;
146 }
147
148 *(int *)arg1 = (int)tt;
149 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1);
150 return 0;
151 }
152
153 #if SYSCTL_SKMEM
154 int tcp_keepinit = TCPTV_KEEP_INIT;
155 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
156 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
157 &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit),
158 sysctl_msec_to_ticks, "I", "");
159
160 int tcp_keepidle = TCPTV_KEEP_IDLE;
161 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
162 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
163 &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle),
164 sysctl_msec_to_ticks, "I", "");
165
166 int tcp_keepintvl = TCPTV_KEEPINTVL;
167 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
168 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
169 &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl),
170 sysctl_msec_to_ticks, "I", "");
171
172 SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt,
173 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
174 int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive");
175
176 int tcp_msl = TCPTV_MSL;
177 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
178 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
179 &tcp_msl, offsetof(skmem_sysctl, tcp.msl),
180 sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
181 #else /* SYSCTL_SKMEM */
182 int tcp_keepinit;
183 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
184 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
185 &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", "");
186
187 int tcp_keepidle;
188 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
189 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
190 &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", "");
191
192 int tcp_keepintvl;
193 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
194 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
195 &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
196
197 int tcp_keepcnt;
198 SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt,
199 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
200 &tcp_keepcnt, 0, "number of times to repeat keepalive");
201
202 int tcp_msl;
203 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
204 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
205 &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
206 #endif /* SYSCTL_SKMEM */
207
208 /*
209 * Avoid DoS with connections half-closed in TIME_WAIT_2
210 */
211 int tcp_fin_timeout = TCPTV_FINWAIT2;
212
213 static int
214 sysctl_tcp_fin_timeout SYSCTL_HANDLER_ARGS
215 {
216 #pragma unused(arg2)
217 int error;
218 int value = tcp_fin_timeout;
219
220 error = sysctl_handle_int(oidp, &value, 0, req);
221 if (error != 0 || req->newptr == USER_ADDR_NULL) {
222 return error;
223 }
224
225 if (value == -1) {
226 /* Reset to default value */
227 value = TCPTV_FINWAIT2;
228 } else {
229 /* Convert from milliseconds */
230 long big_value = value * TCP_RETRANSHZ / 1000;
231
232 if (big_value < 0 || big_value > INT_MAX) {
233 return EINVAL;
234 }
235 value = (int)big_value;
236 }
237 tcp_fin_timeout = value;
238 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, value);
239 return 0;
240 }
241
242 #if SYSCTL_SKMEM
243 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
244 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
245 &tcp_fin_timeout, offsetof(skmem_sysctl, tcp.fin_timeout),
246 sysctl_tcp_fin_timeout, "I", "");
247 #else /* SYSCTL_SKMEM */
248 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
249 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
250 &tcp_fin_timeout, 0,
251 sysctl_tcp_fin_timeout, "I", "");
252 #endif /* SYSCTL_SKMEM */
253
254 /*
255 * Avoid DoS via TCP Robustness in Persist Condition
256 * (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt)
257 * by allowing a system wide maximum persistence timeout value when in
258 * Zero Window Probe mode.
259 *
260 * Expressed in milliseconds to be consistent without timeout related
261 * values, the TCP socket option is in seconds.
262 */
263 #if SYSCTL_SKMEM
264 u_int32_t tcp_max_persist_timeout = 0;
265 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
266 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
267 &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout),
268 sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP");
269 #else /* SYSCTL_SKMEM */
270 u_int32_t tcp_max_persist_timeout = 0;
271 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
272 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
273 &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I",
274 "Maximum persistence timeout for ZWP");
275 #endif /* SYSCTL_SKMEM */
276
277 SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive,
278 CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0,
279 "Assume SO_KEEPALIVE on all TCP connections");
280
281 /*
282 * This parameter determines how long the timer list will stay in fast or
283 * quick mode even though all connections are idle. In this state, the
284 * timer will run more frequently anticipating new data.
285 */
286 SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax,
287 CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax,
288 TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode");
289
290 /*
291 * See tcp_syn_backoff[] for interval values between SYN retransmits;
292 * the value set below defines the number of retransmits, before we
293 * disable the timestamp and window scaling options during subsequent
294 * SYN retransmits. Setting it to 0 disables the dropping off of those
295 * two options.
296 */
297 SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres,
298 CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres,
299 10, "Number of retransmitted SYNs before disabling RFC 1323 "
300 "options on local connections");
301
302 static int tcp_timer_advanced = 0;
303 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced,
304 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_timer_advanced, 0,
305 "Number of times one of the timers was advanced");
306
307 static int tcp_resched_timerlist = 0;
308 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist,
309 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0,
310 "Number of times timer list was rescheduled as part of processing a packet");
311
312 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection,
313 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1,
314 "Path MTU Discovery Black Hole Detection");
315
316 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss,
317 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200,
318 "Path MTU Discovery Black Hole Detection lowered MSS");
319
320 #if (DEBUG || DEVELOPMENT)
321 int tcp_probe_if_fix_port = 0;
322 SYSCTL_INT(_net_inet_tcp, OID_AUTO, probe_if_fix_port,
323 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
324 &tcp_probe_if_fix_port, 0, "");
325 #endif /* (DEBUG || DEVELOPMENT) */
326
327 static u_int32_t tcp_mss_rec_medium = 1200;
328 static u_int32_t tcp_mss_rec_low = 512;
329
330 #define TCP_REPORT_STATS_INTERVAL 43200 /* 12 hours, in seconds */
331 int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL;
332
333 /* performed garbage collection of "used" sockets */
334 static boolean_t tcp_gc_done = FALSE;
335
336 /* max idle probes */
337 int tcp_maxpersistidle = TCPTV_KEEP_IDLE;
338
339 /*
340 * TCP delack timer is set to 100 ms. Since the processing of timer list
341 * in fast mode will happen no faster than 100 ms, the delayed ack timer
342 * will fire some where between 100 and 200 ms.
343 */
344 int tcp_delack = TCP_RETRANSHZ / 10;
345
346 #if MPTCP
347 /*
348 * MP_JOIN retransmission of 3rd ACK will be every 500 msecs without backoff
349 */
350 int tcp_jack_rxmt = TCP_RETRANSHZ / 2;
351 #endif /* MPTCP */
352
353 static boolean_t tcp_itimer_done = FALSE;
354
355 static void tcp_remove_timer(struct tcpcb *tp);
356 static void tcp_sched_timerlist(uint32_t offset);
357 static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode,
358 u_int16_t probe_if_index);
359 static inline void tcp_set_lotimer_index(struct tcpcb *);
360 __private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp);
361 static inline void tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp);
362 __private_extern__ void tcp_report_stats(void);
363
364 static u_int64_t tcp_last_report_time;
365
366 /*
367 * Structure to store previously reported stats so that we can send
368 * incremental changes in each report interval.
369 */
370 struct tcp_last_report_stats {
371 u_int32_t tcps_connattempt;
372 u_int32_t tcps_accepts;
373 u_int32_t tcps_ecn_client_setup;
374 u_int32_t tcps_ecn_server_setup;
375 u_int32_t tcps_ecn_client_success;
376 u_int32_t tcps_ecn_server_success;
377 u_int32_t tcps_ecn_not_supported;
378 u_int32_t tcps_ecn_lost_syn;
379 u_int32_t tcps_ecn_lost_synack;
380 u_int32_t tcps_ecn_recv_ce;
381 u_int32_t tcps_ecn_recv_ece;
382 u_int32_t tcps_ecn_sent_ece;
383 u_int32_t tcps_ecn_conn_recv_ce;
384 u_int32_t tcps_ecn_conn_recv_ece;
385 u_int32_t tcps_ecn_conn_plnoce;
386 u_int32_t tcps_ecn_conn_pl_ce;
387 u_int32_t tcps_ecn_conn_nopl_ce;
388 u_int32_t tcps_ecn_fallback_synloss;
389 u_int32_t tcps_ecn_fallback_reorder;
390 u_int32_t tcps_ecn_fallback_ce;
391
392 /* TFO-related statistics */
393 u_int32_t tcps_tfo_syn_data_rcv;
394 u_int32_t tcps_tfo_cookie_req_rcv;
395 u_int32_t tcps_tfo_cookie_sent;
396 u_int32_t tcps_tfo_cookie_invalid;
397 u_int32_t tcps_tfo_cookie_req;
398 u_int32_t tcps_tfo_cookie_rcv;
399 u_int32_t tcps_tfo_syn_data_sent;
400 u_int32_t tcps_tfo_syn_data_acked;
401 u_int32_t tcps_tfo_syn_loss;
402 u_int32_t tcps_tfo_blackhole;
403 u_int32_t tcps_tfo_cookie_wrong;
404 u_int32_t tcps_tfo_no_cookie_rcv;
405 u_int32_t tcps_tfo_heuristics_disable;
406 u_int32_t tcps_tfo_sndblackhole;
407
408 /* MPTCP-related statistics */
409 u_int32_t tcps_mptcp_handover_attempt;
410 u_int32_t tcps_mptcp_interactive_attempt;
411 u_int32_t tcps_mptcp_aggregate_attempt;
412 u_int32_t tcps_mptcp_fp_handover_attempt;
413 u_int32_t tcps_mptcp_fp_interactive_attempt;
414 u_int32_t tcps_mptcp_fp_aggregate_attempt;
415 u_int32_t tcps_mptcp_heuristic_fallback;
416 u_int32_t tcps_mptcp_fp_heuristic_fallback;
417 u_int32_t tcps_mptcp_handover_success_wifi;
418 u_int32_t tcps_mptcp_handover_success_cell;
419 u_int32_t tcps_mptcp_interactive_success;
420 u_int32_t tcps_mptcp_aggregate_success;
421 u_int32_t tcps_mptcp_fp_handover_success_wifi;
422 u_int32_t tcps_mptcp_fp_handover_success_cell;
423 u_int32_t tcps_mptcp_fp_interactive_success;
424 u_int32_t tcps_mptcp_fp_aggregate_success;
425 u_int32_t tcps_mptcp_handover_cell_from_wifi;
426 u_int32_t tcps_mptcp_handover_wifi_from_cell;
427 u_int32_t tcps_mptcp_interactive_cell_from_wifi;
428 u_int64_t tcps_mptcp_handover_cell_bytes;
429 u_int64_t tcps_mptcp_interactive_cell_bytes;
430 u_int64_t tcps_mptcp_aggregate_cell_bytes;
431 u_int64_t tcps_mptcp_handover_all_bytes;
432 u_int64_t tcps_mptcp_interactive_all_bytes;
433 u_int64_t tcps_mptcp_aggregate_all_bytes;
434 u_int32_t tcps_mptcp_back_to_wifi;
435 u_int32_t tcps_mptcp_wifi_proxy;
436 u_int32_t tcps_mptcp_cell_proxy;
437 u_int32_t tcps_mptcp_triggered_cell;
438 };
439
440
441 /* Returns true if the timer is on the timer list */
442 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
443
444 /* Run the TCP timerlist atleast once every hour */
445 #define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ)
446
447
448 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
449 static boolean_t tcp_garbage_collect(struct inpcb *, int);
450
451 #define TIMERENTRY_TO_TP(te) (__unsafe_forge_single(struct tcpcb *, ((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next))))
452
453 #define VERIFY_NEXT_LINK(elm, field) do { \
454 if (LIST_NEXT((elm),field) != NULL && \
455 LIST_NEXT((elm),field)->field.le_prev != \
456 &((elm)->field.le_next)) \
457 panic("Bad link elm %p next->prev != elm", (elm)); \
458 } while(0)
459
460 #define VERIFY_PREV_LINK(elm, field) do { \
461 if (*(elm)->field.le_prev != (elm)) \
462 panic("Bad link elm %p prev->next != elm", (elm)); \
463 } while(0)
464
465 #define TCP_SET_TIMER_MODE(mode, i) do { \
466 if (IS_TIMER_HZ_10MS(i)) \
467 (mode) |= TCP_TIMERLIST_10MS_MODE; \
468 else if (IS_TIMER_HZ_100MS(i)) \
469 (mode) |= TCP_TIMERLIST_100MS_MODE; \
470 else \
471 (mode) |= TCP_TIMERLIST_500MS_MODE; \
472 } while(0)
473
474 #if (DEVELOPMENT || DEBUG)
475 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_medium,
476 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_medium, 0,
477 "Medium MSS based on recommendation in link status report");
478 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_low,
479 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_low, 0,
480 "Low MSS based on recommendation in link status report");
481
482 static int32_t tcp_change_mss_recommended = 0;
483 static int
484 sysctl_change_mss_recommended SYSCTL_HANDLER_ARGS
485 {
486 #pragma unused(oidp, arg1, arg2)
487 int i, err = 0, changed = 0;
488 struct ifnet *ifp;
489 struct if_link_status ifsr;
490 struct if_cellular_status_v1 *new_cell_sr;
491 err = sysctl_io_number(req, tcp_change_mss_recommended,
492 sizeof(int32_t), &i, &changed);
493 if (changed) {
494 if (i < 0 || i > UINT16_MAX) {
495 return EINVAL;
496 }
497 ifnet_head_lock_shared();
498 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
499 if (IFNET_IS_CELLULAR(ifp)) {
500 bzero(&ifsr, sizeof(ifsr));
501 new_cell_sr = &ifsr.ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
502 ifsr.ifsr_version = IF_CELLULAR_STATUS_REPORT_CURRENT_VERSION;
503 ifsr.ifsr_len = sizeof(*new_cell_sr);
504
505 /* Set MSS recommended */
506 new_cell_sr->valid_bitmask |= IF_CELL_UL_MSS_RECOMMENDED_VALID;
507 new_cell_sr->mss_recommended = (uint16_t)i;
508 err = ifnet_link_status_report(ifp, new_cell_sr, sizeof(new_cell_sr));
509 if (err == 0) {
510 tcp_change_mss_recommended = i;
511 } else {
512 break;
513 }
514 }
515 }
516 ifnet_head_done();
517 }
518 return err;
519 }
520
521 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, change_mss_recommended,
522 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_change_mss_recommended,
523 0, sysctl_change_mss_recommended, "IU", "Change MSS recommended");
524
525 SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval,
526 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0,
527 "Report stats interval");
528 #endif /* (DEVELOPMENT || DEBUG) */
529
530 /*
531 * Macro to compare two timers. If there is a reset of the sign bit,
532 * it is safe to assume that the timer has wrapped around. By doing
533 * signed comparision, we take care of wrap around such that the value
534 * with the sign bit reset is actually ahead of the other.
535 */
536 inline int32_t
timer_diff(uint32_t t1,uint32_t toff1,uint32_t t2,uint32_t toff2)537 timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2)
538 {
539 return (int32_t)((t1 + toff1) - (t2 + toff2));
540 }
541
542 /*
543 * Add to tcp timewait list, delay is given in milliseconds.
544 */
545 static void
add_to_time_wait_locked(struct tcpcb * tp,uint32_t delay)546 add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay)
547 {
548 struct inpcbinfo *pcbinfo = &tcbinfo;
549 struct inpcb *inp = tp->t_inpcb;
550 uint32_t timer;
551
552 /* pcb list should be locked when we get here */
553 LCK_RW_ASSERT(&pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE);
554
555 /* We may get here multiple times, so check */
556 if (!(inp->inp_flags2 & INP2_TIMEWAIT)) {
557 pcbinfo->ipi_twcount++;
558 inp->inp_flags2 |= INP2_TIMEWAIT;
559
560 /* Remove from global inp list */
561 LIST_REMOVE(inp, inp_list);
562 } else {
563 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
564 }
565
566 /* Compute the time at which this socket can be closed */
567 timer = tcp_now + delay;
568
569 /* We will use the TCPT_2MSL timer for tracking this delay */
570
571 if (TIMER_IS_ON_LIST(tp)) {
572 tcp_remove_timer(tp);
573 }
574 tp->t_timer[TCPT_2MSL] = timer;
575
576 TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry);
577 }
578
579 void
add_to_time_wait(struct tcpcb * tp,uint32_t delay)580 add_to_time_wait(struct tcpcb *tp, uint32_t delay)
581 {
582 if (tp->t_inpcb->inp_socket->so_options & SO_NOWAKEFROMSLEEP) {
583 socket_post_kev_msg_closed(tp->t_inpcb->inp_socket);
584 }
585
586 tcp_del_fsw_flow(tp);
587
588 /* 19182803: Notify nstat that connection is closing before waiting. */
589 nstat_pcb_detach(tp->t_inpcb);
590
591 #if CONTENT_FILTER
592 if ((tp->t_inpcb->inp_socket->so_flags & SOF_CONTENT_FILTER) != 0) {
593 /* If filter present, allow filter to finish processing all queued up data before adding to time wait queue */
594 (void) cfil_sock_tcp_add_time_wait(tp->t_inpcb->inp_socket);
595 } else
596 #endif /* CONTENT_FILTER */
597 {
598 add_to_time_wait_now(tp, delay);
599 }
600 }
601
602 void
add_to_time_wait_now(struct tcpcb * tp,uint32_t delay)603 add_to_time_wait_now(struct tcpcb *tp, uint32_t delay)
604 {
605 struct inpcbinfo *pcbinfo = &tcbinfo;
606
607 if (!lck_rw_try_lock_exclusive(&pcbinfo->ipi_lock)) {
608 socket_unlock(tp->t_inpcb->inp_socket, 0);
609 lck_rw_lock_exclusive(&pcbinfo->ipi_lock);
610 socket_lock(tp->t_inpcb->inp_socket, 0);
611 }
612 add_to_time_wait_locked(tp, delay);
613 lck_rw_done(&pcbinfo->ipi_lock);
614
615 inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY);
616 }
617
618 /* If this is on time wait queue, remove it. */
619 void
tcp_remove_from_time_wait(struct inpcb * inp)620 tcp_remove_from_time_wait(struct inpcb *inp)
621 {
622 struct tcpcb *tp = intotcpcb(inp);
623 if (inp->inp_flags2 & INP2_TIMEWAIT) {
624 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
625 }
626 }
627
628 static boolean_t
tcp_garbage_collect(struct inpcb * inp,int istimewait)629 tcp_garbage_collect(struct inpcb *inp, int istimewait)
630 {
631 boolean_t active = FALSE;
632 struct socket *so, *mp_so = NULL;
633 struct tcpcb *tp;
634
635 so = inp->inp_socket;
636 tp = intotcpcb(inp);
637
638 if (so->so_flags & SOF_MP_SUBFLOW) {
639 mp_so = mptetoso(tptomptp(tp)->mpt_mpte);
640 if (!socket_try_lock(mp_so)) {
641 mp_so = NULL;
642 active = TRUE;
643 goto out;
644 }
645 if (mpsotomppcb(mp_so)->mpp_inside > 0) {
646 os_log(mptcp_log_handle, "%s - %lx: Still inside %d usecount %d\n", __func__,
647 (unsigned long)VM_KERNEL_ADDRPERM(mpsotompte(mp_so)),
648 mpsotomppcb(mp_so)->mpp_inside,
649 mp_so->so_usecount);
650 socket_unlock(mp_so, 0);
651 mp_so = NULL;
652 active = TRUE;
653 goto out;
654 }
655 /* We call socket_unlock with refcount further below */
656 mp_so->so_usecount++;
657 tptomptp(tp)->mpt_mpte->mpte_mppcb->mpp_inside++;
658 }
659
660 /*
661 * Skip if still in use or busy; it would have been more efficient
662 * if we were to test so_usecount against 0, but this isn't possible
663 * due to the current implementation of tcp_dropdropablreq() where
664 * overflow sockets that are eligible for garbage collection have
665 * their usecounts set to 1.
666 */
667 if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) {
668 active = TRUE;
669 goto out;
670 }
671
672 /* Check again under the lock */
673 if (so->so_usecount > 1) {
674 if (inp->inp_wantcnt == WNT_STOPUSING) {
675 active = TRUE;
676 }
677 lck_mtx_unlock(&inp->inpcb_mtx);
678 goto out;
679 }
680
681 if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) &&
682 tp->t_state != TCPS_CLOSED) {
683 /* Become a regular mutex */
684 lck_mtx_convert_spin(&inp->inpcb_mtx);
685 tcp_close(tp);
686 }
687
688 /*
689 * Overflowed socket dropped from the listening queue? Do this
690 * only if we are called to clean up the time wait slots, since
691 * tcp_dropdropablreq() considers a socket to have been fully
692 * dropped after add_to_time_wait() is finished.
693 * Also handle the case of connections getting closed by the peer
694 * while in the queue as seen with rdar://6422317
695 *
696 */
697 if (so->so_usecount == 1 &&
698 ((istimewait && (so->so_flags & SOF_OVERFLOW)) ||
699 ((tp != NULL) && (tp->t_state == TCPS_CLOSED) &&
700 (so->so_head != NULL) &&
701 ((so->so_state & (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE)) ==
702 (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE))))) {
703 if (inp->inp_state != INPCB_STATE_DEAD) {
704 /* Become a regular mutex */
705 lck_mtx_convert_spin(&inp->inpcb_mtx);
706 if (SOCK_CHECK_DOM(so, PF_INET6)) {
707 in6_pcbdetach(inp);
708 } else {
709 in_pcbdetach(inp);
710 }
711 }
712 VERIFY(so->so_usecount > 0);
713 so->so_usecount--;
714 if (inp->inp_wantcnt == WNT_STOPUSING) {
715 active = TRUE;
716 }
717 lck_mtx_unlock(&inp->inpcb_mtx);
718 goto out;
719 } else if (inp->inp_wantcnt != WNT_STOPUSING) {
720 lck_mtx_unlock(&inp->inpcb_mtx);
721 active = FALSE;
722 goto out;
723 }
724
725 /*
726 * We get here because the PCB is no longer searchable
727 * (WNT_STOPUSING); detach (if needed) and dispose if it is dead
728 * (usecount is 0). This covers all cases, including overflow
729 * sockets and those that are considered as "embryonic",
730 * i.e. created by sonewconn() in TCP input path, and have
731 * not yet been committed. For the former, we reduce the usecount
732 * to 0 as done by the code above. For the latter, the usecount
733 * would have reduced to 0 as part calling soabort() when the
734 * socket is dropped at the end of tcp_input().
735 */
736 if (so->so_usecount == 0) {
737 DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp,
738 struct tcpcb *, tp, int32_t, TCPS_CLOSED);
739 /* Become a regular mutex */
740 lck_mtx_convert_spin(&inp->inpcb_mtx);
741
742 /*
743 * If this tp still happens to be on the timer list,
744 * take it out
745 */
746 if (TIMER_IS_ON_LIST(tp)) {
747 tcp_remove_timer(tp);
748 }
749
750 if (inp->inp_state != INPCB_STATE_DEAD) {
751 if (SOCK_CHECK_DOM(so, PF_INET6)) {
752 in6_pcbdetach(inp);
753 } else {
754 in_pcbdetach(inp);
755 }
756 }
757
758 if (mp_so) {
759 mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub);
760
761 /* so is now unlinked from mp_so - let's drop the lock */
762 socket_unlock(mp_so, 1);
763 mp_so = NULL;
764 }
765
766 in_pcbdispose(inp);
767 active = FALSE;
768 goto out;
769 }
770
771 lck_mtx_unlock(&inp->inpcb_mtx);
772 active = TRUE;
773
774 out:
775 if (mp_so) {
776 socket_unlock(mp_so, 1);
777 }
778
779 return active;
780 }
781
782 /*
783 * TCP garbage collector callback (inpcb_timer_func_t).
784 *
785 * Returns the number of pcbs that will need to be gc-ed soon,
786 * returnining > 0 will keep timer active.
787 */
788 void
tcp_gc(struct inpcbinfo * ipi)789 tcp_gc(struct inpcbinfo *ipi)
790 {
791 struct inpcb *inp, *nxt;
792 struct tcpcb *tw_tp, *tw_ntp;
793 #if KDEBUG
794 static int tws_checked = 0;
795 #endif
796
797 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0);
798
799 /*
800 * Update tcp_now here as it may get used while
801 * processing the slow timer.
802 */
803 calculate_tcp_clock();
804
805 /*
806 * Garbage collect socket/tcpcb: We need to acquire the list lock
807 * exclusively to do this
808 */
809
810 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
811 /* don't sweat it this time; cleanup was done last time */
812 if (tcp_gc_done == TRUE) {
813 tcp_gc_done = FALSE;
814 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END,
815 tws_checked, cur_tw_slot, 0, 0, 0);
816 /* Lock upgrade failed, give up this round */
817 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
818 return;
819 }
820 /* Upgrade failed, lost lock now take it again exclusive */
821 lck_rw_lock_exclusive(&ipi->ipi_lock);
822 }
823 tcp_gc_done = TRUE;
824
825 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
826 if (tcp_garbage_collect(inp, 0)) {
827 os_atomic_inc(&ipi->ipi_gc_req.intimer_fast, relaxed);
828 }
829 }
830
831 /* Now cleanup the time wait ones */
832 TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) {
833 /*
834 * We check the timestamp here without holding the
835 * socket lock for better performance. If there are
836 * any pcbs in time-wait, the timer will get rescheduled.
837 * Hence some error in this check can be tolerated.
838 *
839 * Sometimes a socket on time-wait queue can be closed if
840 * 2MSL timer expired but the application still has a
841 * usecount on it.
842 */
843 if (tw_tp->t_state == TCPS_CLOSED ||
844 TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) {
845 if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) {
846 os_atomic_inc(&ipi->ipi_gc_req.intimer_lazy, relaxed);
847 }
848 }
849 }
850
851 /* take into account pcbs that are still in time_wait_slots */
852 os_atomic_add(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount, relaxed);
853
854 lck_rw_done(&ipi->ipi_lock);
855
856 /* Clean up the socache while we are here */
857 if (so_cache_timer()) {
858 os_atomic_inc(&ipi->ipi_gc_req.intimer_lazy, relaxed);
859 }
860
861 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked,
862 cur_tw_slot, 0, 0, 0);
863
864 return;
865 }
866
867 /*
868 * Cancel all timers for TCP tp.
869 */
870 void
tcp_canceltimers(struct tcpcb * tp)871 tcp_canceltimers(struct tcpcb *tp)
872 {
873 int i;
874
875 tcp_remove_timer(tp);
876 for (i = 0; i < TCPT_NTIMERS; i++) {
877 tp->t_timer[i] = 0;
878 }
879 tp->tentry.timer_start = tcp_now;
880 tp->tentry.index = TCPT_NONE;
881 }
882
883 int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =
884 { 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 };
885
886 int tcp_backoff[TCP_MAXRXTSHIFT + 1] =
887 { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
888
889 static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */
890
891 void
tcp_rexmt_save_state(struct tcpcb * tp)892 tcp_rexmt_save_state(struct tcpcb *tp)
893 {
894 u_int32_t fsize;
895 if (TSTMP_SUPPORTED(tp)) {
896 /*
897 * Since timestamps are supported on the connection,
898 * we can do recovery as described in rfc 4015.
899 */
900 fsize = tp->snd_max - tp->snd_una;
901 tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh);
902 tp->snd_recover_prev = tp->snd_recover;
903 } else {
904 /*
905 * Timestamp option is not supported on this connection.
906 * Record ssthresh and cwnd so they can
907 * be recovered if this turns out to be a "bad" retransmit.
908 * A retransmit is considered "bad" if an ACK for this
909 * segment is received within RTT/2 interval; the assumption
910 * here is that the ACK was already in flight. See
911 * "On Estimating End-to-End Network Path Properties" by
912 * Allman and Paxson for more details.
913 */
914 tp->snd_cwnd_prev = tp->snd_cwnd;
915 tp->snd_ssthresh_prev = tp->snd_ssthresh;
916 tp->snd_recover_prev = tp->snd_recover;
917 if (IN_FASTRECOVERY(tp)) {
918 tp->t_flags |= TF_WASFRECOVERY;
919 } else {
920 tp->t_flags &= ~TF_WASFRECOVERY;
921 }
922 }
923 tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2;
924 tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT);
925 tp->t_flagsext &= ~(TF_RECOMPUTE_RTT);
926 }
927
928 /*
929 * Revert to the older segment size if there is an indication that PMTU
930 * blackhole detection was not needed.
931 */
932 void
tcp_pmtud_revert_segment_size(struct tcpcb * tp)933 tcp_pmtud_revert_segment_size(struct tcpcb *tp)
934 {
935 int32_t optlen;
936
937 VERIFY(tp->t_pmtud_saved_maxopd > 0);
938 tp->t_flags |= TF_PMTUD;
939 tp->t_flags &= ~TF_BLACKHOLE;
940 optlen = tp->t_maxopd - tp->t_maxseg;
941 tp->t_maxopd = tp->t_pmtud_saved_maxopd;
942 tp->t_maxseg = tp->t_maxopd - optlen;
943
944 /*
945 * Reset the slow-start flight size as it
946 * may depend on the new MSS
947 */
948 if (CC_ALGO(tp)->cwnd_init != NULL) {
949 CC_ALGO(tp)->cwnd_init(tp);
950 }
951
952 if (TCP_USE_RLEDBAT(tp, tp->t_inpcb->inp_socket) &&
953 tcp_cc_rledbat.rwnd_init != NULL) {
954 tcp_cc_rledbat.rwnd_init(tp);
955 }
956
957 tp->t_pmtud_start_ts = 0;
958 tcpstat.tcps_pmtudbh_reverted++;
959
960 /* change MSS according to recommendation, if there was one */
961 tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL);
962 }
963
964 static uint32_t
tcp_pmtud_black_holed_next_mss(struct tcpcb * tp)965 tcp_pmtud_black_holed_next_mss(struct tcpcb *tp)
966 {
967 /* Reduce the MSS to intermediary value */
968 if (tp->t_maxopd > tcp_pmtud_black_hole_mss) {
969 return tcp_pmtud_black_hole_mss;
970 } else {
971 if (tp->t_inpcb->inp_vflag & INP_IPV4) {
972 return tcp_mssdflt;
973 } else {
974 return tcp_v6mssdflt;
975 }
976 }
977 }
978
979 /*
980 * Send a packet designed to force a response
981 * if the peer is up and reachable:
982 * either an ACK if the connection is still alive,
983 * or an RST if the peer has closed the connection
984 * due to timeout or reboot.
985 * Using sequence number tp->snd_una-1
986 * causes the transmitted zero-length segment
987 * to lie outside the receive window;
988 * by the protocol spec, this requires the
989 * correspondent TCP to respond.
990 */
991 static bool
tcp_send_keep_alive(struct tcpcb * tp)992 tcp_send_keep_alive(struct tcpcb *tp)
993 {
994 struct tcptemp *__single t_template;
995 struct mbuf *__single m;
996
997 tcpstat.tcps_keepprobe++;
998 t_template = tcp_maketemplate(tp, &m);
999 if (t_template != NULL) {
1000 struct inpcb *inp = tp->t_inpcb;
1001 struct tcp_respond_args tra;
1002
1003 bzero(&tra, sizeof(tra));
1004 tra.nocell = INP_NO_CELLULAR(inp) ? 1 : 0;
1005 tra.noexpensive = INP_NO_EXPENSIVE(inp) ? 1 : 0;
1006 tra.noconstrained = INP_NO_CONSTRAINED(inp) ? 1 : 0;
1007 tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp) ? 1 : 0;
1008 tra.intcoproc_allowed = INP_INTCOPROC_ALLOWED(inp) ? 1 : 0;
1009 tra.management_allowed = INP_MANAGEMENT_ALLOWED(inp) ? 1 : 0;
1010 tra.keep_alive = 1;
1011 if (tp->t_inpcb->inp_flags & INP_BOUND_IF) {
1012 tra.ifscope = tp->t_inpcb->inp_boundifp->if_index;
1013 } else {
1014 tra.ifscope = IFSCOPE_NONE;
1015 }
1016 tcp_respond(tp, t_template->tt_ipgen, sizeof(t_template->tt_ipgen),
1017 &t_template->tt_t, (struct mbuf *)NULL,
1018 tp->rcv_nxt, tp->snd_una - 1, 0, &tra);
1019 (void) m_free(m);
1020 return true;
1021 } else {
1022 return false;
1023 }
1024 }
1025
1026 /*
1027 * TCP timer processing.
1028 */
1029 struct tcpcb *
tcp_timers(struct tcpcb * tp,int timer)1030 tcp_timers(struct tcpcb *tp, int timer)
1031 {
1032 int32_t rexmt, optlen = 0, idle_time = 0;
1033 struct socket *so;
1034 u_int64_t accsleep_ms;
1035 u_int64_t last_sleep_ms = 0;
1036 struct ifnet *outifp = tp->t_inpcb->inp_last_outifp;
1037
1038 so = tp->t_inpcb->inp_socket;
1039 idle_time = tcp_now - tp->t_rcvtime;
1040
1041 switch (timer) {
1042 /*
1043 * 2 MSL timeout in shutdown went off. If we're closed but
1044 * still waiting for peer to close and connection has been idle
1045 * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2,
1046 * delete connection control block.
1047 * Otherwise, (this case shouldn't happen) check again in a bit
1048 * we keep the socket in the main list in that case.
1049 */
1050 case TCPT_2MSL:
1051 tcp_free_sackholes(tp);
1052 if (tp->t_state != TCPS_TIME_WAIT &&
1053 tp->t_state != TCPS_FIN_WAIT_2 &&
1054 ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) {
1055 tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp,
1056 (u_int32_t)TCP_CONN_KEEPINTVL(tp));
1057 } else {
1058 if (tp->t_state == TCPS_FIN_WAIT_2) {
1059 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1060 "FIN wait timeout drop");
1061 tcpstat.tcps_fin_timeout_drops++;
1062 tp = tcp_drop(tp, 0);
1063 } else {
1064 tp = tcp_close(tp);
1065 }
1066 return tp;
1067 }
1068 break;
1069
1070 /*
1071 * Retransmission timer went off. Message has not
1072 * been acked within retransmit interval. Back off
1073 * to a longer retransmit interval and retransmit one segment.
1074 */
1075 case TCPT_REXMT:
1076 absolutetime_to_nanoseconds(mach_absolutetime_asleep,
1077 &accsleep_ms);
1078 accsleep_ms = accsleep_ms / 1000000UL;
1079 if (accsleep_ms > tp->t_accsleep_ms) {
1080 last_sleep_ms = accsleep_ms - tp->t_accsleep_ms;
1081 }
1082 /*
1083 * Drop a connection in the retransmit timer
1084 * 1. If we have retransmitted more than TCP_MAXRXTSHIFT
1085 * times
1086 * 2. If the time spent in this retransmission episode is
1087 * more than the time limit set with TCP_RXT_CONNDROPTIME
1088 * socket option
1089 * 3. If TCP_RXT_FINDROP socket option was set and
1090 * we have already retransmitted the FIN 3 times without
1091 * receiving an ack
1092 */
1093 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT ||
1094 (tp->t_rxt_conndroptime > 0 && tp->t_rxtstart > 0 &&
1095 (tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime) ||
1096 ((tp->t_flagsext & TF_RXTFINDROP) != 0 &&
1097 (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) ||
1098 (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) {
1099 if (tp->t_state == TCPS_ESTABLISHED &&
1100 tp->t_rxt_minimum_timeout > 0) {
1101 /*
1102 * Avoid dropping a connection if minimum
1103 * timeout is set and that time did not
1104 * pass. We will retry sending
1105 * retransmissions at the maximum interval
1106 */
1107 if (TSTMP_LT(tcp_now, (tp->t_rxtstart +
1108 tp->t_rxt_minimum_timeout))) {
1109 tp->t_rxtshift = TCP_MAXRXTSHIFT - 1;
1110 goto retransmit_packet;
1111 }
1112 }
1113 if ((tp->t_flagsext & TF_RXTFINDROP) != 0) {
1114 tcpstat.tcps_rxtfindrop++;
1115 } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) {
1116 tcpstat.tcps_drop_after_sleep++;
1117 } else {
1118 tcpstat.tcps_timeoutdrop++;
1119 }
1120 if (tp->t_rxtshift >= TCP_MAXRXTSHIFT) {
1121 if (TCP_ECN_ENABLED(tp)) {
1122 INP_INC_IFNET_STAT(tp->t_inpcb,
1123 ecn_on.rxmit_drop);
1124 } else {
1125 INP_INC_IFNET_STAT(tp->t_inpcb,
1126 ecn_off.rxmit_drop);
1127 }
1128 }
1129 tp->t_rxtshift = TCP_MAXRXTSHIFT;
1130 soevent(so,
1131 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1132
1133 if (TCP_ECN_ENABLED(tp) &&
1134 tp->t_state == TCPS_ESTABLISHED) {
1135 tcp_heuristic_ecn_droprxmt(tp);
1136 }
1137
1138 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1139 "retransmission timeout drop");
1140 tp = tcp_drop(tp, tp->t_softerror ?
1141 tp->t_softerror : ETIMEDOUT);
1142
1143 break;
1144 }
1145 retransmit_packet:
1146 tcpstat.tcps_rexmttimeo++;
1147 tp->t_accsleep_ms = accsleep_ms;
1148
1149 if (tp->t_rxtshift == 1 &&
1150 tp->t_state == TCPS_ESTABLISHED) {
1151 /* Set the time at which retransmission started. */
1152 tp->t_rxtstart = tcp_now;
1153
1154 /*
1155 * if this is the first retransmit timeout, save
1156 * the state so that we can recover if the timeout
1157 * is spurious.
1158 */
1159 tcp_rexmt_save_state(tp);
1160 tcp_ccdbg_trace(tp, NULL, TCP_CC_FIRST_REXMT);
1161 }
1162 #if MPTCP
1163 if ((tp->t_rxtshift >= mptcp_fail_thresh) &&
1164 (tp->t_state == TCPS_ESTABLISHED) &&
1165 (tp->t_mpflags & TMPF_MPTCP_TRUE)) {
1166 mptcp_act_on_txfail(so);
1167 }
1168
1169 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1170 (so->so_flags & SOF_MP_SUBFLOW)) {
1171 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1172
1173 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER ||
1174 mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
1175 mptcp_check_subflows_and_add(mpte);
1176 }
1177 }
1178 #endif /* MPTCP */
1179
1180 if (tp->t_adaptive_wtimo > 0 &&
1181 tp->t_rxtshift > tp->t_adaptive_wtimo &&
1182 TCPS_HAVEESTABLISHED(tp->t_state)) {
1183 /* Send an event to the application */
1184 soevent(so,
1185 (SO_FILT_HINT_LOCKED |
1186 SO_FILT_HINT_ADAPTIVE_WTIMO));
1187 }
1188
1189 /*
1190 * If this is a retransmit timeout after PTO, the PTO
1191 * was not effective
1192 */
1193 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1194 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1195 tcpstat.tcps_rto_after_pto++;
1196 }
1197
1198 if (tp->t_flagsext & TF_DELAY_RECOVERY) {
1199 /*
1200 * Retransmit timer fired before entering recovery
1201 * on a connection with packet re-ordering. This
1202 * suggests that the reordering metrics computed
1203 * are not accurate.
1204 */
1205 tp->t_reorderwin = 0;
1206 tp->t_timer[TCPT_DELAYFR] = 0;
1207 tp->t_flagsext &= ~(TF_DELAY_RECOVERY);
1208 }
1209
1210 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1211 tp->t_state == TCPS_SYN_RECEIVED) {
1212 tcp_disable_tfo(tp);
1213 }
1214
1215 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1216 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1217 (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
1218 !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
1219 ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||
1220 tp->t_rxtshift > 4)) {
1221 /*
1222 * For regular retransmissions, a first one is being
1223 * done for tail-loss probe.
1224 * Thus, if rxtshift > 1, this means we have sent the segment
1225 * a total of 3 times.
1226 *
1227 * If we are in SYN-SENT state, then there is no tail-loss
1228 * probe thus we have to let rxtshift go up to 3.
1229 */
1230 tcp_heuristic_tfo_middlebox(tp);
1231
1232 so->so_error = ENODATA;
1233 soevent(so,
1234 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1235 sorwakeup(so);
1236 sowwakeup(so);
1237
1238 tp->t_tfo_stats |= TFO_S_SEND_BLACKHOLE;
1239 tcpstat.tcps_tfo_sndblackhole++;
1240 }
1241
1242 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1243 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1244 (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&
1245 tp->t_rxtshift > 3) {
1246 if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
1247 tcp_heuristic_tfo_middlebox(tp);
1248
1249 so->so_error = ENODATA;
1250 soevent(so,
1251 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1252 sorwakeup(so);
1253 sowwakeup(so);
1254 }
1255 }
1256
1257 if (tp->t_state == TCPS_SYN_SENT) {
1258 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_SYNRMXT) != 0 &&
1259 if_link_heuristics_enabled(outifp)) {
1260 IF_TCP_STATINC(outifp, linkheur_synrxmt);
1261 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1262 } else {
1263 rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift];
1264 }
1265 tp->t_stat.synrxtshift = tp->t_rxtshift;
1266 tp->t_stat.rxmitsyns++;
1267
1268 /* When retransmitting, disable TFO */
1269 if (TFO_ENABLED(tp) &&
1270 !(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE)) {
1271 tcp_disable_tfo(tp);
1272 tp->t_tfo_flags |= TFO_F_SYN_LOSS;
1273 }
1274 } else {
1275 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1276 }
1277 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX,
1278 TCP_ADD_REXMTSLOP(tp));
1279
1280 tcp_set_link_heur_rtomin(tp, outifp);
1281 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1282
1283 TCP_LOG_RTT_INFO(tp);
1284
1285 if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb)) {
1286 goto fc_output;
1287 }
1288
1289 tcp_free_sackholes(tp);
1290 if (TCP_RACK_ENABLED(tp)) {
1291 tcp_segs_clear_sacked(tp);
1292 tcp_rack_loss_on_rto(tp, true);
1293 }
1294 /*
1295 * Check for potential Path MTU Discovery Black Hole
1296 */
1297 if (tcp_pmtud_black_hole_detect &&
1298 !(tp->t_flagsext & TF_NOBLACKHOLE_DETECTION) &&
1299 (tp->t_state == TCPS_ESTABLISHED)) {
1300 if ((tp->t_flags & TF_PMTUD) &&
1301 tp->t_pmtud_lastseg_size > tcp_pmtud_black_holed_next_mss(tp) &&
1302 tp->t_rxtshift == 2) {
1303 /*
1304 * Enter Path MTU Black-hole Detection mechanism:
1305 * - Disable Path MTU Discovery (IP "DF" bit).
1306 * - Reduce MTU to lower value than what we
1307 * negotiated with the peer.
1308 */
1309 /* Disable Path MTU Discovery for now */
1310 tp->t_flags &= ~TF_PMTUD;
1311 /* Record that we may have found a black hole */
1312 tp->t_flags |= TF_BLACKHOLE;
1313 optlen = tp->t_maxopd - tp->t_maxseg;
1314 /* Keep track of previous MSS */
1315 tp->t_pmtud_saved_maxopd = tp->t_maxopd;
1316 tp->t_pmtud_start_ts = tcp_now;
1317 if (tp->t_pmtud_start_ts == 0) {
1318 tp->t_pmtud_start_ts++;
1319 }
1320 /* Reduce the MSS to intermediary value */
1321 tp->t_maxopd = tcp_pmtud_black_holed_next_mss(tp);
1322 tp->t_maxseg = tp->t_maxopd - optlen;
1323
1324 /*
1325 * Reset the slow-start flight size
1326 * as it may depend on the new MSS
1327 */
1328 if (CC_ALGO(tp)->cwnd_init != NULL) {
1329 CC_ALGO(tp)->cwnd_init(tp);
1330 }
1331 tp->snd_cwnd = tp->t_maxseg;
1332
1333 if (TCP_USE_RLEDBAT(tp, so) &&
1334 tcp_cc_rledbat.rwnd_init != NULL) {
1335 tcp_cc_rledbat.rwnd_init(tp);
1336 }
1337 }
1338 /*
1339 * If further retransmissions are still
1340 * unsuccessful with a lowered MTU, maybe this
1341 * isn't a Black Hole and we restore the previous
1342 * MSS and blackhole detection flags.
1343 */
1344 else {
1345 if ((tp->t_flags & TF_BLACKHOLE) &&
1346 (tp->t_rxtshift > 4)) {
1347 tcp_pmtud_revert_segment_size(tp);
1348 tp->snd_cwnd = tp->t_maxseg;
1349 }
1350 }
1351 }
1352
1353 /*
1354 * Disable rfc1323 and rfc1644 if we haven't got any
1355 * response to our SYN (after we reach the threshold)
1356 * to work-around some broken terminal servers (most of
1357 * which have hopefully been retired) that have bad VJ
1358 * header compression code which trashes TCP segments
1359 * containing unknown-to-them TCP options.
1360 * Do this only on non-local connections.
1361 */
1362 if (tp->t_state == TCPS_SYN_SENT &&
1363 tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) {
1364 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP);
1365 }
1366
1367 /*
1368 * If losing, let the lower level know and try for
1369 * a better route. Also, if we backed off this far,
1370 * our srtt estimate is probably bogus. Clobber it
1371 * so we'll take the next rtt measurement as our srtt;
1372 * move the current srtt into rttvar to keep the current
1373 * retransmit times until then.
1374 */
1375 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
1376 if (!(tp->t_inpcb->inp_vflag & INP_IPV4)) {
1377 in6_losing(tp->t_inpcb);
1378 } else {
1379 in_losing(tp->t_inpcb);
1380 }
1381 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
1382 tp->t_srtt = 0;
1383 }
1384 tp->snd_nxt = tp->snd_una;
1385 /*
1386 * Note: We overload snd_recover to function also as the
1387 * snd_last variable described in RFC 2582
1388 */
1389 tp->snd_recover = tp->snd_max;
1390 /*
1391 * Force a segment to be sent.
1392 */
1393 tp->t_flags |= TF_ACKNOW;
1394
1395 /*
1396 * If timing a segment in this window, stop the timer
1397 * except when we are in connecting states on cellular
1398 * interfaces
1399 */
1400 if (tp->t_state >= TCPS_ESTABLISHED || (outifp != NULL &&
1401 IFNET_IS_CELLULAR(outifp) == false)) {
1402 tp->t_rtttime = 0;
1403 }
1404
1405 if (!IN_FASTRECOVERY(tp) && tp->t_rxtshift == 1) {
1406 tcpstat.tcps_tailloss_rto++;
1407 }
1408
1409 /*
1410 * RFC 5681 says: when a TCP sender detects segment loss
1411 * using retransmit timer and the given segment has already
1412 * been retransmitted by way of the retransmission timer at
1413 * least once, the value of ssthresh is held constant
1414 */
1415 if (tp->t_rxtshift == 1 &&
1416 CC_ALGO(tp)->after_timeout != NULL) {
1417 CC_ALGO(tp)->after_timeout(tp);
1418 /*
1419 * CWR notifications are to be sent on new data
1420 * right after Fast Retransmits and ECE
1421 * notification receipts.
1422 */
1423 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1424 tp->ecn_flags |= TE_SENDCWR;
1425 }
1426 }
1427
1428 EXIT_FASTRECOVERY(tp);
1429
1430 /* Exit cwnd non validated phase */
1431 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
1432
1433
1434 fc_output:
1435 tcp_ccdbg_trace(tp, NULL, TCP_CC_REXMT_TIMEOUT);
1436
1437 (void) tcp_output(tp);
1438 break;
1439
1440 /*
1441 * Persistance timer into zero window.
1442 * Force a byte to be output, if possible.
1443 */
1444 case TCPT_PERSIST:
1445 tcpstat.tcps_persisttimeo++;
1446 /*
1447 * Hack: if the peer is dead/unreachable, we do not
1448 * time out if the window is closed. After a full
1449 * backoff, drop the connection if the idle time
1450 * (no responses to probes) reaches the maximum
1451 * backoff that we would use if retransmitting.
1452 *
1453 * Drop the connection if we reached the maximum allowed time for
1454 * Zero Window Probes without a non-zero update from the peer.
1455 * See rdar://5805356
1456 */
1457 if ((tp->t_rxtshift == TCP_MAXRXTSHIFT &&
1458 (idle_time >= tcp_maxpersistidle ||
1459 idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) ||
1460 ((tp->t_persist_stop != 0) &&
1461 TSTMP_LEQ(tp->t_persist_stop, tcp_now))) {
1462 TCP_LOG_DROP_PCB(NULL, NULL, tp, false, "persist timeout drop");
1463 tcpstat.tcps_persistdrop++;
1464 soevent(so,
1465 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1466 tp = tcp_drop(tp, ETIMEDOUT);
1467 break;
1468 }
1469 tcp_setpersist(tp);
1470 tp->t_flagsext |= TF_FORCE;
1471 (void) tcp_output(tp);
1472 tp->t_flagsext &= ~TF_FORCE;
1473 break;
1474
1475 /*
1476 * Keep-alive timer went off; send something
1477 * or drop connection if idle for too long.
1478 */
1479 case TCPT_KEEP:
1480 #if FLOW_DIVERT
1481 if (tp->t_inpcb->inp_socket->so_flags & SOF_FLOW_DIVERT) {
1482 break;
1483 }
1484 #endif /* FLOW_DIVERT */
1485
1486 tcpstat.tcps_keeptimeo++;
1487 #if MPTCP
1488 /*
1489 * Regular TCP connections do not send keepalives after closing
1490 * MPTCP must not also, after sending Data FINs.
1491 */
1492 struct mptcb *mp_tp = tptomptp(tp);
1493 if ((tp->t_mpflags & TMPF_MPTCP_TRUE) &&
1494 (tp->t_state > TCPS_ESTABLISHED)) {
1495 goto dropit;
1496 } else if (mp_tp != NULL) {
1497 if ((mptcp_ok_to_keepalive(mp_tp) == 0)) {
1498 goto dropit;
1499 }
1500 }
1501 #endif /* MPTCP */
1502 if (tp->t_state < TCPS_ESTABLISHED) {
1503 goto dropit;
1504 }
1505 if ((always_keepalive ||
1506 (tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) ||
1507 (tp->t_flagsext & TF_DETECT_READSTALL) ||
1508 (tp->t_tfo_probe_state == TFO_PROBE_PROBING)) &&
1509 (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
1510 if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp)) {
1511 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1512 "keep alive timeout drop");
1513 goto dropit;
1514 }
1515
1516 if (tcp_send_keep_alive(tp)) {
1517 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1518 tp->t_rtimo_probes++;
1519 }
1520
1521 TCP_LOG_KEEP_ALIVE(tp, idle_time);
1522 }
1523
1524 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1525 TCP_CONN_KEEPINTVL(tp));
1526 } else {
1527 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1528 TCP_CONN_KEEPIDLE(tp));
1529 }
1530 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1531 bool reenable_probe = false;
1532 /*
1533 * The keep alive packets sent to detect a read
1534 * stall did not get a response from the
1535 * peer. Generate more keep-alives to confirm this.
1536 * If the number of probes sent reaches the limit,
1537 * generate an event.
1538 */
1539 if (tp->t_adaptive_rtimo > 0) {
1540 if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) {
1541 /* Generate an event */
1542 soevent(so,
1543 (SO_FILT_HINT_LOCKED |
1544 SO_FILT_HINT_ADAPTIVE_RTIMO));
1545 tcp_keepalive_reset(tp);
1546 } else {
1547 reenable_probe = true;
1548 }
1549 } else if (outifp != NULL &&
1550 (outifp->if_eflags & IFEF_PROBE_CONNECTIVITY) &&
1551 tp->t_rtimo_probes <= TCP_CONNECTIVITY_PROBES_MAX) {
1552 reenable_probe = true;
1553 } else {
1554 tp->t_flagsext &= ~TF_DETECT_READSTALL;
1555 }
1556 if (reenable_probe) {
1557 int ind = min(tp->t_rtimo_probes,
1558 TCP_MAXRXTSHIFT);
1559 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(
1560 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp));
1561 }
1562 }
1563 if (tp->t_tfo_probe_state == TFO_PROBE_PROBING) {
1564 int ind;
1565
1566 tp->t_tfo_probes++;
1567 ind = min(tp->t_tfo_probes, TCP_MAXRXTSHIFT);
1568
1569 /*
1570 * We take the minimum among the time set by true
1571 * keepalive (see above) and the backoff'd RTO. That
1572 * way we backoff in case of packet-loss but will never
1573 * timeout slower than regular keepalive due to the
1574 * backing off.
1575 */
1576 tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START(
1577 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)),
1578 tp->t_timer[TCPT_KEEP]);
1579 } else if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1580 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1581 tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) {
1582 /* Still no data! Let's assume a TFO-error and err out... */
1583 tcp_heuristic_tfo_middlebox(tp);
1584
1585 so->so_error = ENODATA;
1586 soevent(so,
1587 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1588 sorwakeup(so);
1589 tp->t_tfo_stats |= TFO_S_RECV_BLACKHOLE;
1590 tcpstat.tcps_tfo_blackhole++;
1591 }
1592 break;
1593 case TCPT_DELACK:
1594 if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) {
1595 tp->t_flags &= ~TF_DELACK;
1596 tp->t_timer[TCPT_DELACK] = 0;
1597 tp->t_flags |= TF_ACKNOW;
1598
1599 /*
1600 * If delayed ack timer fired while stretching
1601 * acks, count the number of times the streaming
1602 * detection was not correct. If this exceeds a
1603 * threshold, disable strech ack on this
1604 * connection
1605 *
1606 * Also, go back to acking every other packet.
1607 */
1608 if ((tp->t_flags & TF_STRETCHACK)) {
1609 if (tp->t_unacksegs > 1 &&
1610 tp->t_unacksegs < maxseg_unacked) {
1611 tp->t_stretchack_delayed++;
1612 }
1613
1614 if (tp->t_stretchack_delayed >
1615 TCP_STRETCHACK_DELAY_THRESHOLD) {
1616 tp->t_flagsext |= TF_DISABLE_STRETCHACK;
1617 /*
1618 * Note the time at which stretch
1619 * ack was disabled automatically
1620 */
1621 tp->rcv_nostrack_ts = tcp_now;
1622 tcpstat.tcps_nostretchack++;
1623 tp->t_stretchack_delayed = 0;
1624 tp->rcv_nostrack_pkts = 0;
1625 }
1626 tcp_reset_stretch_ack(tp);
1627 }
1628 tp->t_forced_acks = TCP_FORCED_ACKS_COUNT;
1629
1630 /*
1631 * If we are measuring inter packet arrival jitter
1632 * for throttling a connection, this delayed ack
1633 * might be the reason for accumulating some
1634 * jitter. So let's restart the measurement.
1635 */
1636 CLEAR_IAJ_STATE(tp);
1637
1638 tcpstat.tcps_delack++;
1639 tp->t_stat.delayed_acks_sent++;
1640 (void) tcp_output(tp);
1641 }
1642 break;
1643
1644 #if MPTCP
1645 case TCPT_JACK_RXMT:
1646 if ((tp->t_state == TCPS_ESTABLISHED) &&
1647 (tp->t_mpflags & TMPF_PREESTABLISHED) &&
1648 (tp->t_mpflags & TMPF_JOINED_FLOW)) {
1649 if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) {
1650 tcpstat.tcps_timeoutdrop++;
1651 soevent(so,
1652 (SO_FILT_HINT_LOCKED |
1653 SO_FILT_HINT_TIMEOUT));
1654 tp = tcp_drop(tp, tp->t_softerror ?
1655 tp->t_softerror : ETIMEDOUT);
1656 break;
1657 }
1658 tcpstat.tcps_join_rxmts++;
1659 tp->t_mpflags |= TMPF_SND_JACK;
1660 tp->t_flags |= TF_ACKNOW;
1661
1662 /*
1663 * No backoff is implemented for simplicity for this
1664 * corner case.
1665 */
1666 (void) tcp_output(tp);
1667 }
1668 break;
1669 case TCPT_CELLICON:
1670 {
1671 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1672
1673 tp->t_timer[TCPT_CELLICON] = 0;
1674
1675 if (mpte->mpte_cellicon_increments == 0) {
1676 /* Cell-icon not set by this connection */
1677 break;
1678 }
1679
1680 if (TSTMP_LT(mpte->mpte_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, tcp_now)) {
1681 mptcp_unset_cellicon(mpte, NULL, 1);
1682 }
1683
1684 if (mpte->mpte_cellicon_increments) {
1685 tp->t_timer[TCPT_CELLICON] = OFFSET_FROM_START(tp, MPTCP_CELLICON_TOGGLE_RATE);
1686 }
1687
1688 break;
1689 }
1690 #endif /* MPTCP */
1691
1692 case TCPT_PTO:
1693 {
1694 int32_t ret = 0;
1695
1696 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1697 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1698 }
1699 /*
1700 * Check if the connection is in the right state to
1701 * send a probe
1702 */
1703 if ((tp->t_state != TCPS_ESTABLISHED ||
1704 tp->t_rxtshift > 0 ||
1705 tp->snd_max == tp->snd_una ||
1706 !SACK_ENABLED(tp) || IN_FASTRECOVERY(tp)) &&
1707 !(tp->t_flagsext & TF_IF_PROBING)) {
1708 break;
1709 }
1710
1711 /*
1712 * When the interface state is changed explicitly reset the retransmission
1713 * timer state for both SYN and data packets because we do not want to
1714 * wait unnecessarily or timeout too quickly if the link characteristics
1715 * have changed drastically
1716 */
1717 if (tp->t_flagsext & TF_IF_PROBING) {
1718 tp->t_rxtshift = 0;
1719 if (tp->t_state == TCPS_SYN_SENT) {
1720 tp->t_stat.synrxtshift = tp->t_rxtshift;
1721 }
1722 /*
1723 * Reset to the the default RTO
1724 */
1725 tp->t_srtt = TCPTV_SRTTBASE;
1726 tp->t_rttvar =
1727 ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
1728 tp->t_rttmin = TCPTV_REXMTMIN;
1729 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
1730 tp->t_rttmin, TCPTV_REXMTMAX, TCP_ADD_REXMTSLOP(tp));
1731 TCP_LOG_RTT_INFO(tp);
1732 }
1733
1734 if (tp->t_state == TCPS_SYN_SENT) {
1735 /*
1736 * The PTO for SYN_SENT reinitializes TCP as if it was a fresh
1737 * connection attempt
1738 */
1739 tp->snd_nxt = tp->snd_una;
1740 /*
1741 * Note: We overload snd_recover to function also as the
1742 * snd_last variable described in RFC 2582
1743 */
1744 tp->snd_recover = tp->snd_max;
1745 /*
1746 * Force a segment to be sent.
1747 */
1748 tp->t_flags |= TF_ACKNOW;
1749
1750 /* If timing a segment in this window, stop the timer */
1751 tp->t_rtttime = 0;
1752
1753 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1754 } else {
1755 int32_t snd_len;
1756
1757 /*
1758 * If there is no new data to send or if the
1759 * connection is limited by receive window then
1760 * retransmit the last segment, otherwise send
1761 * new data.
1762 */
1763 snd_len = min(so->so_snd.sb_cc, tp->snd_wnd)
1764 - (tp->snd_max - tp->snd_una);
1765 if (snd_len > 0) {
1766 tp->snd_nxt = tp->snd_max;
1767 tp->t_flagsext &= ~TF_TLP_IS_RETRANS;
1768 } else {
1769 snd_len = min((tp->snd_max - tp->snd_una),
1770 tp->t_maxseg);
1771 tp->snd_nxt = tp->snd_max - snd_len;
1772 tp->t_flagsext |= TF_TLP_IS_RETRANS;
1773 }
1774 }
1775
1776 tcpstat.tcps_pto++;
1777 if (tp->t_flagsext & TF_IF_PROBING) {
1778 tcpstat.tcps_probe_if++;
1779 }
1780
1781 /* If timing a segment in this window, stop the timer */
1782 tp->t_rtttime = 0;
1783 /* Note that tail loss probe is being sent. Exclude IF probe */
1784 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1785 tp->t_flagsext |= TF_SENT_TLPROBE;
1786 tp->t_tlpstart = tcp_now;
1787 }
1788
1789 tp->snd_cwnd += tp->t_maxseg;
1790 /*
1791 * When tail-loss-probe fires, we reset the RTO timer, because
1792 * a probe just got sent, so we are good to push out the timer.
1793 *
1794 * Set to 0 to ensure that tcp_output() will reschedule it
1795 */
1796 tp->t_timer[TCPT_REXMT] = 0;
1797 ret = tcp_output(tp);
1798
1799 #if (DEBUG || DEVELOPMENT)
1800 if ((tp->t_flagsext & TF_IF_PROBING) &&
1801 ((IFNET_IS_COMPANION_LINK(tp->t_inpcb->inp_last_outifp)) ||
1802 tp->t_state == TCPS_SYN_SENT)) {
1803 if (ret == 0 && tcp_probe_if_fix_port > 0 &&
1804 tcp_probe_if_fix_port <= IPPORT_HILASTAUTO) {
1805 tp->t_timer[TCPT_REXMT] = 0;
1806 tcp_set_lotimer_index(tp);
1807 }
1808
1809 os_log(OS_LOG_DEFAULT,
1810 "%s: sent %s probe for %u > %u on interface %s"
1811 " (%u) %s(%d)",
1812 __func__,
1813 tp->t_state == TCPS_SYN_SENT ? "SYN" : "data",
1814 ntohs(tp->t_inpcb->inp_lport),
1815 ntohs(tp->t_inpcb->inp_fport),
1816 if_name(tp->t_inpcb->inp_last_outifp),
1817 tp->t_inpcb->inp_last_outifp->if_index,
1818 ret == 0 ? "succeeded" :"failed", ret);
1819 }
1820 #endif /* DEBUG || DEVELOPMENT */
1821
1822 /*
1823 * When there is data (or a SYN) to send, the above call to
1824 * tcp_output() should have armed either the REXMT or the
1825 * PERSIST timer. If it didn't, something is wrong and this
1826 * connection would idle around forever. Let's make sure that
1827 * at least the REXMT timer is set.
1828 */
1829 if (tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0 &&
1830 (tp->t_inpcb->inp_socket->so_snd.sb_cc != 0 || tp->t_state == TCPS_SYN_SENT ||
1831 tp->t_state == TCPS_SYN_RECEIVED)) {
1832 tcp_set_link_heur_rtomin(tp, outifp);
1833 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1834
1835 os_log(OS_LOG_DEFAULT,
1836 "%s: tcp_output() returned %u with retransmission timer disabled "
1837 "for %u > %u in state %d, reset timer to %d",
1838 __func__, ret,
1839 ntohs(tp->t_inpcb->inp_lport),
1840 ntohs(tp->t_inpcb->inp_fport),
1841 tp->t_state,
1842 tp->t_timer[TCPT_REXMT]);
1843
1844 tcp_check_timer_state(tp);
1845 }
1846 tp->snd_cwnd -= tp->t_maxseg;
1847
1848 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1849 tp->t_tlphighrxt = tp->snd_nxt;
1850 tp->t_tlphightrxt_persist = tp->snd_nxt;
1851 }
1852 break;
1853 }
1854 case TCPT_DELAYFR:
1855 tp->t_flagsext &= ~TF_DELAY_RECOVERY;
1856
1857 /*
1858 * Don't do anything if one of the following is true:
1859 * - the connection is already in recovery
1860 * - sequence until snd_recover has been acknowledged.
1861 * - retransmit timeout has fired
1862 */
1863 if (IN_FASTRECOVERY(tp) ||
1864 SEQ_GEQ(tp->snd_una, tp->snd_recover) ||
1865 tp->t_rxtshift > 0) {
1866 break;
1867 }
1868
1869 VERIFY(SACK_ENABLED(tp));
1870 tcp_rexmt_save_state(tp);
1871 if (CC_ALGO(tp)->pre_fr != NULL) {
1872 CC_ALGO(tp)->pre_fr(tp);
1873 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1874 tp->ecn_flags |= TE_SENDCWR;
1875 }
1876 }
1877 ENTER_FASTRECOVERY(tp);
1878
1879 tp->t_timer[TCPT_REXMT] = 0;
1880 tcpstat.tcps_sack_recovery_episode++;
1881 tp->t_sack_recovery_episode++;
1882 tp->snd_cwnd = tp->t_maxseg;
1883 tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY);
1884 (void) tcp_output(tp);
1885 break;
1886
1887 dropit:
1888 tcpstat.tcps_keepdrops++;
1889 soevent(so,
1890 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1891 tp = tcp_drop(tp, ETIMEDOUT);
1892 break;
1893 case TCPT_REORDER:
1894 if (TCP_RACK_ENABLED(tp)) {
1895 tcp_rack_reordering_timeout(tp, 0);
1896 }
1897 break;
1898 }
1899 return tp;
1900 }
1901
1902 /* Remove a timer entry from timer list */
1903 void
tcp_remove_timer(struct tcpcb * tp)1904 tcp_remove_timer(struct tcpcb *tp)
1905 {
1906 struct tcptimerlist *listp = &tcp_timer_list;
1907
1908 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
1909 if (!(TIMER_IS_ON_LIST(tp))) {
1910 return;
1911 }
1912 lck_mtx_lock(&listp->mtx);
1913
1914 if (listp->next_te != NULL && listp->next_te == &tp->tentry) {
1915 listp->next_te = LIST_NEXT(&tp->tentry, le);
1916 }
1917
1918 LIST_REMOVE(&tp->tentry, le);
1919 tp->t_flags &= ~(TF_TIMER_ONLIST);
1920
1921 listp->entries--;
1922
1923 tp->tentry.le.le_next = NULL;
1924 tp->tentry.le.le_prev = NULL;
1925
1926 lck_mtx_unlock(&listp->mtx);
1927 }
1928
1929 /*
1930 * Function to check if the timerlist needs to be rescheduled to run
1931 * the timer entry correctly. Basically, this is to check if we can avoid
1932 * taking the list lock.
1933 */
1934
1935 static boolean_t
need_to_resched_timerlist(u_int32_t runtime,u_int16_t mode)1936 need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode)
1937 {
1938 struct tcptimerlist *listp = &tcp_timer_list;
1939 int32_t diff;
1940
1941 /*
1942 * If the list is being processed then the state of the list is
1943 * in flux. In this case always acquire the lock and set the state
1944 * correctly.
1945 */
1946 if (listp->running) {
1947 return TRUE;
1948 }
1949
1950 if (!listp->scheduled) {
1951 return TRUE;
1952 }
1953
1954 diff = timer_diff(listp->runtime, 0, runtime, 0);
1955 if (diff <= 0) {
1956 /* The list is going to run before this timer */
1957 return FALSE;
1958 } else {
1959 if (mode & TCP_TIMERLIST_10MS_MODE) {
1960 if (diff <= TCP_TIMER_10MS_QUANTUM) {
1961 return FALSE;
1962 }
1963 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
1964 if (diff <= TCP_TIMER_100MS_QUANTUM) {
1965 return FALSE;
1966 }
1967 } else {
1968 if (diff <= TCP_TIMER_500MS_QUANTUM) {
1969 return FALSE;
1970 }
1971 }
1972 }
1973 return TRUE;
1974 }
1975
1976 void
tcp_sched_timerlist(uint32_t offset)1977 tcp_sched_timerlist(uint32_t offset)
1978 {
1979 uint64_t deadline = 0;
1980 struct tcptimerlist *listp = &tcp_timer_list;
1981
1982 LCK_MTX_ASSERT(&listp->mtx, LCK_MTX_ASSERT_OWNED);
1983
1984 offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);
1985 listp->runtime = tcp_now + offset;
1986 listp->schedtime = tcp_now;
1987 if (listp->runtime == 0) {
1988 listp->runtime++;
1989 offset++;
1990 }
1991
1992 clock_interval_to_deadline(offset, USEC_PER_SEC, &deadline);
1993
1994 thread_call_enter_delayed(listp->call, deadline);
1995 listp->scheduled = TRUE;
1996 }
1997
1998 /*
1999 * Function to run the timers for a connection.
2000 *
2001 * Returns the offset of next timer to be run for this connection which
2002 * can be used to reschedule the timerlist.
2003 *
2004 * te_mode is an out parameter that indicates the modes of active
2005 * timers for this connection.
2006 */
2007 u_int32_t
tcp_run_conn_timer(struct tcpcb * tp,u_int16_t * te_mode,u_int16_t probe_if_index)2008 tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode,
2009 u_int16_t probe_if_index)
2010 {
2011 struct socket *so;
2012 u_int16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE;
2013 u_int32_t timer_val, offset = 0, lo_timer = 0;
2014 int32_t diff;
2015 boolean_t needtorun[TCPT_NTIMERS];
2016 int count = 0;
2017
2018 VERIFY(tp != NULL);
2019 bzero(needtorun, sizeof(needtorun));
2020 *te_mode = 0;
2021
2022 socket_lock(tp->t_inpcb->inp_socket, 1);
2023
2024 so = tp->t_inpcb->inp_socket;
2025 /* Release the want count on inp */
2026 if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1)
2027 == WNT_STOPUSING) {
2028 if (TIMER_IS_ON_LIST(tp)) {
2029 tcp_remove_timer(tp);
2030 }
2031
2032 /* Looks like the TCP connection got closed while we
2033 * were waiting for the lock.. Done
2034 */
2035 goto done;
2036 }
2037
2038 /*
2039 * If this connection is over an interface that needs to
2040 * be probed, send probe packets to reinitiate communication.
2041 */
2042 if (TCP_IF_STATE_CHANGED(tp, probe_if_index)) {
2043 tp->t_flagsext |= TF_IF_PROBING;
2044 tcp_timers(tp, TCPT_PTO);
2045 tp->t_timer[TCPT_PTO] = 0;
2046 tp->t_flagsext &= ~TF_IF_PROBING;
2047 }
2048
2049 /*
2050 * Since the timer thread needs to wait for tcp lock, it may race
2051 * with another thread that can cancel or reschedule the timer
2052 * that is about to run. Check if we need to run anything.
2053 */
2054 if ((index = tp->tentry.index) == TCPT_NONE) {
2055 goto done;
2056 }
2057
2058 timer_val = tp->t_timer[index];
2059
2060 diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
2061 if (diff > 0) {
2062 if (tp->tentry.index != TCPT_NONE) {
2063 offset = diff;
2064 *(te_mode) = tp->tentry.mode;
2065 }
2066 goto done;
2067 }
2068
2069 tp->t_timer[index] = 0;
2070 if (timer_val > 0) {
2071 tp = tcp_timers(tp, index);
2072 if (tp == NULL) {
2073 goto done;
2074 }
2075 }
2076
2077 /*
2078 * Check if there are any other timers that need to be run.
2079 * While doing it, adjust the timer values wrt tcp_now.
2080 */
2081 tp->tentry.mode = 0;
2082 for (i = 0; i < TCPT_NTIMERS; ++i) {
2083 if (tp->t_timer[i] != 0) {
2084 diff = timer_diff(tp->tentry.timer_start,
2085 tp->t_timer[i], tcp_now, 0);
2086 if (diff <= 0) {
2087 needtorun[i] = TRUE;
2088 count++;
2089 } else {
2090 tp->t_timer[i] = diff;
2091 needtorun[i] = FALSE;
2092 if (lo_timer == 0 || diff < lo_timer) {
2093 lo_timer = diff;
2094 lo_index = i;
2095 }
2096 TCP_SET_TIMER_MODE(tp->tentry.mode, i);
2097 }
2098 }
2099 }
2100
2101 tp->tentry.timer_start = tcp_now;
2102 tp->tentry.index = lo_index;
2103 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2104
2105 if (tp->tentry.index != TCPT_NONE) {
2106 tp->tentry.runtime = tp->tentry.timer_start +
2107 tp->t_timer[tp->tentry.index];
2108 if (tp->tentry.runtime == 0) {
2109 tp->tentry.runtime++;
2110 }
2111 }
2112
2113 if (count > 0) {
2114 /* run any other timers outstanding at this time. */
2115 for (i = 0; i < TCPT_NTIMERS; ++i) {
2116 if (needtorun[i]) {
2117 tp->t_timer[i] = 0;
2118 tp = tcp_timers(tp, i);
2119 if (tp == NULL) {
2120 offset = 0;
2121 *(te_mode) = 0;
2122 goto done;
2123 }
2124 }
2125 }
2126 tcp_set_lotimer_index(tp);
2127 }
2128
2129 if (tp->tentry.index < TCPT_NONE) {
2130 offset = tp->t_timer[tp->tentry.index];
2131 *(te_mode) = tp->tentry.mode;
2132 }
2133
2134 done:
2135 if (tp != NULL && tp->tentry.index == TCPT_NONE) {
2136 tcp_remove_timer(tp);
2137 offset = 0;
2138 }
2139
2140 socket_unlock(so, 1);
2141 return offset;
2142 }
2143
2144 void
tcp_run_timerlist(void * arg1,void * arg2)2145 tcp_run_timerlist(void * arg1, void * arg2)
2146 {
2147 #pragma unused(arg1, arg2)
2148 struct tcptimerentry *te, *__single next_te;
2149 struct tcptimerlist *__single listp = &tcp_timer_list;
2150 struct tcpcb *__single tp;
2151 uint32_t next_timer = 0; /* offset of the next timer on the list */
2152 u_int16_t te_mode = 0; /* modes of all active timers in a tcpcb */
2153 u_int16_t list_mode = 0; /* cumulative of modes of all tcpcbs */
2154 uint32_t active_count = 0;
2155
2156 calculate_tcp_clock();
2157
2158 lck_mtx_lock(&listp->mtx);
2159
2160 int32_t drift = tcp_now - listp->runtime;
2161 if (drift <= 1) {
2162 tcpstat.tcps_timer_drift_le_1_ms++;
2163 } else if (drift <= 10) {
2164 tcpstat.tcps_timer_drift_le_10_ms++;
2165 } else if (drift <= 20) {
2166 tcpstat.tcps_timer_drift_le_20_ms++;
2167 } else if (drift <= 50) {
2168 tcpstat.tcps_timer_drift_le_50_ms++;
2169 } else if (drift <= 100) {
2170 tcpstat.tcps_timer_drift_le_100_ms++;
2171 } else if (drift <= 200) {
2172 tcpstat.tcps_timer_drift_le_200_ms++;
2173 } else if (drift <= 500) {
2174 tcpstat.tcps_timer_drift_le_500_ms++;
2175 } else if (drift <= 1000) {
2176 tcpstat.tcps_timer_drift_le_1000_ms++;
2177 } else {
2178 tcpstat.tcps_timer_drift_gt_1000_ms++;
2179 }
2180
2181 listp->running = TRUE;
2182
2183 LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
2184 uint32_t offset = 0;
2185 uint32_t runtime = te->runtime;
2186
2187 tp = TIMERENTRY_TO_TP(te);
2188
2189 /*
2190 * An interface probe may need to happen before the previously scheduled runtime
2191 */
2192 if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now) &&
2193 !TCP_IF_STATE_CHANGED(tp, listp->probe_if_index)) {
2194 offset = timer_diff(runtime, 0, tcp_now, 0);
2195 if (next_timer == 0 || offset < next_timer) {
2196 next_timer = offset;
2197 }
2198 list_mode |= te->mode;
2199 continue;
2200 }
2201
2202 /*
2203 * Acquire an inp wantcnt on the inpcb so that the socket
2204 * won't get detached even if tcp_close is called
2205 */
2206 if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0)
2207 == WNT_STOPUSING) {
2208 /*
2209 * Need to take socket lock because it protects
2210 * TIMER_IS_ON_LIST
2211 */
2212 lck_mtx_unlock(&listp->mtx);
2213 socket_lock(tp->t_inpcb->inp_socket, 1);
2214 tcp_remove_timer(tp);
2215 socket_unlock(tp->t_inpcb->inp_socket, 1);
2216 lck_mtx_lock(&listp->mtx);
2217 continue;
2218 }
2219 active_count++;
2220
2221 /*
2222 * Store the next timerentry pointer before releasing the
2223 * list lock. If that entry has to be removed when we
2224 * release the lock, this pointer will be updated to the
2225 * element after that.
2226 */
2227 listp->next_te = next_te;
2228
2229 VERIFY_NEXT_LINK(&tp->tentry, le);
2230 VERIFY_PREV_LINK(&tp->tentry, le);
2231
2232 lck_mtx_unlock(&listp->mtx);
2233
2234 offset = tcp_run_conn_timer(tp, &te_mode,
2235 listp->probe_if_index);
2236
2237 lck_mtx_lock(&listp->mtx);
2238
2239 next_te = listp->next_te;
2240 listp->next_te = NULL;
2241
2242 if (offset > 0 && te_mode != 0) {
2243 list_mode |= te_mode;
2244
2245 if (next_timer == 0 || offset < next_timer) {
2246 next_timer = offset;
2247 }
2248 }
2249 }
2250
2251 if (!LIST_EMPTY(&listp->lhead)) {
2252 uint32_t next_mode = 0;
2253 if ((list_mode & TCP_TIMERLIST_10MS_MODE) ||
2254 (listp->pref_mode & TCP_TIMERLIST_10MS_MODE)) {
2255 next_mode = TCP_TIMERLIST_10MS_MODE;
2256 } else if ((list_mode & TCP_TIMERLIST_100MS_MODE) ||
2257 (listp->pref_mode & TCP_TIMERLIST_100MS_MODE)) {
2258 next_mode = TCP_TIMERLIST_100MS_MODE;
2259 } else {
2260 next_mode = TCP_TIMERLIST_500MS_MODE;
2261 }
2262
2263 if (next_mode != TCP_TIMERLIST_500MS_MODE) {
2264 listp->idleruns = 0;
2265 } else {
2266 /*
2267 * the next required mode is slow mode, but if
2268 * the last one was a faster mode and we did not
2269 * have enough idle runs, repeat the last mode.
2270 *
2271 * We try to keep the timer list in fast mode for
2272 * some idle time in expectation of new data.
2273 */
2274 if (listp->mode != next_mode &&
2275 listp->idleruns < timer_fastmode_idlemax) {
2276 listp->idleruns++;
2277 next_mode = listp->mode;
2278 next_timer = TCP_TIMER_100MS_QUANTUM;
2279 } else {
2280 listp->idleruns = 0;
2281 }
2282 }
2283 listp->mode = next_mode;
2284 if (listp->pref_offset != 0) {
2285 next_timer = min(listp->pref_offset, next_timer);
2286 }
2287
2288 if (listp->mode == TCP_TIMERLIST_500MS_MODE) {
2289 next_timer = max(next_timer,
2290 TCP_TIMER_500MS_QUANTUM);
2291 }
2292
2293 tcp_sched_timerlist(next_timer);
2294 } else {
2295 /*
2296 * No need to reschedule this timer, but always run
2297 * periodically at a much higher granularity.
2298 */
2299 tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);
2300 }
2301
2302 listp->running = FALSE;
2303 listp->pref_mode = 0;
2304 listp->pref_offset = 0;
2305 listp->probe_if_index = 0;
2306
2307 lck_mtx_unlock(&listp->mtx);
2308 }
2309
2310 /*
2311 * Function to check if the timerlist needs to be rescheduled to run this
2312 * connection's timers correctly.
2313 */
2314 void
tcp_sched_timers(struct tcpcb * tp)2315 tcp_sched_timers(struct tcpcb *tp)
2316 {
2317 struct tcptimerentry *te = &tp->tentry;
2318 u_int16_t index = te->index;
2319 u_int16_t mode = te->mode;
2320 struct tcptimerlist *listp = &tcp_timer_list;
2321 int32_t offset = 0;
2322 boolean_t list_locked = FALSE;
2323
2324 if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) {
2325 /* Just return without adding the dead pcb to the list */
2326 if (TIMER_IS_ON_LIST(tp)) {
2327 tcp_remove_timer(tp);
2328 }
2329 return;
2330 }
2331
2332 if (index == TCPT_NONE) {
2333 /* Nothing to run */
2334 tcp_remove_timer(tp);
2335 return;
2336 }
2337
2338 /*
2339 * compute the offset at which the next timer for this connection
2340 * has to run.
2341 */
2342 offset = timer_diff(te->runtime, 0, tcp_now, 0);
2343 if (offset <= 0) {
2344 offset = 1;
2345 tcp_timer_advanced++;
2346 }
2347
2348 if (!TIMER_IS_ON_LIST(tp)) {
2349 if (!list_locked) {
2350 lck_mtx_lock(&listp->mtx);
2351 list_locked = TRUE;
2352 }
2353
2354 if (!TIMER_IS_ON_LIST(tp)) {
2355 LIST_INSERT_HEAD(&listp->lhead, te, le);
2356 tp->t_flags |= TF_TIMER_ONLIST;
2357
2358 listp->entries++;
2359 if (listp->entries > listp->maxentries) {
2360 listp->maxentries = listp->entries;
2361 }
2362
2363 /* if the list is not scheduled, just schedule it */
2364 if (!listp->scheduled) {
2365 goto schedule;
2366 }
2367 }
2368 }
2369
2370 /*
2371 * Timer entry is currently on the list, check if the list needs
2372 * to be rescheduled.
2373 */
2374 if (need_to_resched_timerlist(te->runtime, mode)) {
2375 tcp_resched_timerlist++;
2376
2377 if (!list_locked) {
2378 lck_mtx_lock(&listp->mtx);
2379 list_locked = TRUE;
2380 }
2381
2382 VERIFY_NEXT_LINK(te, le);
2383 VERIFY_PREV_LINK(te, le);
2384
2385 if (listp->running) {
2386 listp->pref_mode |= mode;
2387 if (listp->pref_offset == 0 ||
2388 offset < listp->pref_offset) {
2389 listp->pref_offset = offset;
2390 }
2391 } else {
2392 /*
2393 * The list could have got rescheduled while
2394 * this thread was waiting for the lock
2395 */
2396 if (listp->scheduled) {
2397 int32_t diff;
2398 diff = timer_diff(listp->runtime, 0,
2399 tcp_now, offset);
2400 if (diff <= 0) {
2401 goto done;
2402 } else {
2403 goto schedule;
2404 }
2405 } else {
2406 goto schedule;
2407 }
2408 }
2409 }
2410 goto done;
2411
2412 schedule:
2413 /*
2414 * Since a connection with timers is getting scheduled, the timer
2415 * list moves from idle to active state and that is why idlegen is
2416 * reset
2417 */
2418 if (mode & TCP_TIMERLIST_10MS_MODE) {
2419 listp->mode = TCP_TIMERLIST_10MS_MODE;
2420 listp->idleruns = 0;
2421 offset = min(offset, TCP_TIMER_10MS_QUANTUM);
2422 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
2423 if (listp->mode > TCP_TIMERLIST_100MS_MODE) {
2424 listp->mode = TCP_TIMERLIST_100MS_MODE;
2425 }
2426 listp->idleruns = 0;
2427 offset = min(offset, TCP_TIMER_100MS_QUANTUM);
2428 }
2429 tcp_sched_timerlist(offset);
2430
2431 done:
2432 if (list_locked) {
2433 lck_mtx_unlock(&listp->mtx);
2434 }
2435
2436 return;
2437 }
2438
2439 static inline void
tcp_set_lotimer_index(struct tcpcb * tp)2440 tcp_set_lotimer_index(struct tcpcb *tp)
2441 {
2442 uint16_t i, lo_index = TCPT_NONE, mode = 0;
2443 uint32_t lo_timer = 0;
2444 for (i = 0; i < TCPT_NTIMERS; ++i) {
2445 if (tp->t_timer[i] != 0) {
2446 TCP_SET_TIMER_MODE(mode, i);
2447 if (lo_timer == 0 || tp->t_timer[i] < lo_timer) {
2448 lo_timer = tp->t_timer[i];
2449 lo_index = i;
2450 }
2451 }
2452 }
2453 tp->tentry.index = lo_index;
2454 tp->tentry.mode = mode;
2455 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2456
2457 if (tp->tentry.index != TCPT_NONE) {
2458 tp->tentry.runtime = tp->tentry.timer_start
2459 + tp->t_timer[tp->tentry.index];
2460 if (tp->tentry.runtime == 0) {
2461 tp->tentry.runtime++;
2462 }
2463 }
2464 }
2465
2466 void
tcp_check_timer_state(struct tcpcb * tp)2467 tcp_check_timer_state(struct tcpcb *tp)
2468 {
2469 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
2470
2471 if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) {
2472 return;
2473 }
2474
2475 tcp_set_lotimer_index(tp);
2476
2477 tcp_sched_timers(tp);
2478 return;
2479 }
2480
2481 static inline void
tcp_cumulative_stat(u_int32_t cur,u_int32_t * prev,u_int32_t * dest)2482 tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest)
2483 {
2484 /* handle wrap around */
2485 int32_t diff = (int32_t) (cur - *prev);
2486 if (diff > 0) {
2487 *dest = diff;
2488 } else {
2489 *dest = 0;
2490 }
2491 *prev = cur;
2492 return;
2493 }
2494
2495 static inline void
tcp_cumulative_stat64(u_int64_t cur,u_int64_t * prev,u_int64_t * dest)2496 tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest)
2497 {
2498 /* handle wrap around */
2499 int64_t diff = (int64_t) (cur - *prev);
2500 if (diff > 0) {
2501 *dest = diff;
2502 } else {
2503 *dest = 0;
2504 }
2505 *prev = cur;
2506 return;
2507 }
2508
2509 __private_extern__ void
tcp_report_stats(void)2510 tcp_report_stats(void)
2511 {
2512 struct nstat_sysinfo_data data;
2513 struct sockaddr_in dst;
2514 struct sockaddr_in6 dst6;
2515 struct rtentry *rt = NULL;
2516 static struct tcp_last_report_stats prev;
2517 u_int64_t var, uptime;
2518
2519 #define stat data.u.tcp_stats
2520 if (((uptime = net_uptime()) - tcp_last_report_time) <
2521 tcp_report_stats_interval) {
2522 return;
2523 }
2524
2525 tcp_last_report_time = uptime;
2526
2527 bzero(&data, sizeof(data));
2528 data.flags = NSTAT_SYSINFO_TCP_STATS;
2529
2530 SOCKADDR_ZERO(&dst, sizeof(dst));
2531 dst.sin_len = sizeof(dst);
2532 dst.sin_family = AF_INET;
2533
2534 /* ipv4 avg rtt */
2535 lck_mtx_lock(rnh_lock);
2536 rt = rt_lookup(TRUE, SA(&dst), NULL,
2537 rt_tables[AF_INET], IFSCOPE_NONE);
2538 lck_mtx_unlock(rnh_lock);
2539 if (rt != NULL) {
2540 RT_LOCK(rt);
2541 if (rt_primary_default(rt, rt_key(rt)) &&
2542 rt->rt_stats != NULL) {
2543 stat.ipv4_avgrtt = rt->rt_stats->nstat_avg_rtt;
2544 }
2545 RT_UNLOCK(rt);
2546 rtfree(rt);
2547 rt = NULL;
2548 }
2549
2550 /* ipv6 avg rtt */
2551 SOCKADDR_ZERO(&dst6, sizeof(dst6));
2552 dst6.sin6_len = sizeof(dst6);
2553 dst6.sin6_family = AF_INET6;
2554
2555 lck_mtx_lock(rnh_lock);
2556 rt = rt_lookup(TRUE, SA(&dst6), NULL,
2557 rt_tables[AF_INET6], IFSCOPE_NONE);
2558 lck_mtx_unlock(rnh_lock);
2559 if (rt != NULL) {
2560 RT_LOCK(rt);
2561 if (rt_primary_default(rt, rt_key(rt)) &&
2562 rt->rt_stats != NULL) {
2563 stat.ipv6_avgrtt = rt->rt_stats->nstat_avg_rtt;
2564 }
2565 RT_UNLOCK(rt);
2566 rtfree(rt);
2567 rt = NULL;
2568 }
2569
2570 /* send packet loss rate, shift by 10 for precision */
2571 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_sndrexmitpack > 0) {
2572 var = tcpstat.tcps_sndrexmitpack << 10;
2573 stat.send_plr = (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2574 }
2575
2576 /* recv packet loss rate, shift by 10 for precision */
2577 if (tcpstat.tcps_rcvpack > 0 && tcpstat.tcps_recovered_pkts > 0) {
2578 var = tcpstat.tcps_recovered_pkts << 10;
2579 stat.recv_plr = (uint32_t)((var * 100) / tcpstat.tcps_rcvpack);
2580 }
2581
2582 /* RTO after tail loss, shift by 10 for precision */
2583 if (tcpstat.tcps_sndrexmitpack > 0
2584 && tcpstat.tcps_tailloss_rto > 0) {
2585 var = tcpstat.tcps_tailloss_rto << 10;
2586 stat.send_tlrto_rate =
2587 (uint32_t)((var * 100) / tcpstat.tcps_sndrexmitpack);
2588 }
2589
2590 /* packet reordering */
2591 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) {
2592 var = tcpstat.tcps_reordered_pkts << 10;
2593 stat.send_reorder_rate =
2594 (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2595 }
2596
2597 if (tcp_ecn_outbound == 1) {
2598 stat.ecn_client_enabled = 1;
2599 }
2600 if (tcp_ecn_inbound == 1) {
2601 stat.ecn_server_enabled = 1;
2602 }
2603 tcp_cumulative_stat(tcpstat.tcps_connattempt,
2604 &prev.tcps_connattempt, &stat.connection_attempts);
2605 tcp_cumulative_stat(tcpstat.tcps_accepts,
2606 &prev.tcps_accepts, &stat.connection_accepts);
2607 tcp_cumulative_stat(tcpstat.tcps_ecn_client_setup,
2608 &prev.tcps_ecn_client_setup, &stat.ecn_client_setup);
2609 tcp_cumulative_stat(tcpstat.tcps_ecn_server_setup,
2610 &prev.tcps_ecn_server_setup, &stat.ecn_server_setup);
2611 tcp_cumulative_stat(tcpstat.tcps_ecn_client_success,
2612 &prev.tcps_ecn_client_success, &stat.ecn_client_success);
2613 tcp_cumulative_stat(tcpstat.tcps_ecn_server_success,
2614 &prev.tcps_ecn_server_success, &stat.ecn_server_success);
2615 tcp_cumulative_stat(tcpstat.tcps_ecn_not_supported,
2616 &prev.tcps_ecn_not_supported, &stat.ecn_not_supported);
2617 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_syn,
2618 &prev.tcps_ecn_lost_syn, &stat.ecn_lost_syn);
2619 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_synack,
2620 &prev.tcps_ecn_lost_synack, &stat.ecn_lost_synack);
2621 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ce,
2622 &prev.tcps_ecn_recv_ce, &stat.ecn_recv_ce);
2623 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2624 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2625 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2626 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2627 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2628 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2629 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2630 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2631 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ce,
2632 &prev.tcps_ecn_conn_recv_ce, &stat.ecn_conn_recv_ce);
2633 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ece,
2634 &prev.tcps_ecn_conn_recv_ece, &stat.ecn_conn_recv_ece);
2635 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_plnoce,
2636 &prev.tcps_ecn_conn_plnoce, &stat.ecn_conn_plnoce);
2637 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_pl_ce,
2638 &prev.tcps_ecn_conn_pl_ce, &stat.ecn_conn_pl_ce);
2639 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_nopl_ce,
2640 &prev.tcps_ecn_conn_nopl_ce, &stat.ecn_conn_nopl_ce);
2641 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_synloss,
2642 &prev.tcps_ecn_fallback_synloss, &stat.ecn_fallback_synloss);
2643 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_reorder,
2644 &prev.tcps_ecn_fallback_reorder, &stat.ecn_fallback_reorder);
2645 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_ce,
2646 &prev.tcps_ecn_fallback_ce, &stat.ecn_fallback_ce);
2647 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_rcv,
2648 &prev.tcps_tfo_syn_data_rcv, &stat.tfo_syn_data_rcv);
2649 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req_rcv,
2650 &prev.tcps_tfo_cookie_req_rcv, &stat.tfo_cookie_req_rcv);
2651 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_sent,
2652 &prev.tcps_tfo_cookie_sent, &stat.tfo_cookie_sent);
2653 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_invalid,
2654 &prev.tcps_tfo_cookie_invalid, &stat.tfo_cookie_invalid);
2655 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req,
2656 &prev.tcps_tfo_cookie_req, &stat.tfo_cookie_req);
2657 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_rcv,
2658 &prev.tcps_tfo_cookie_rcv, &stat.tfo_cookie_rcv);
2659 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_sent,
2660 &prev.tcps_tfo_syn_data_sent, &stat.tfo_syn_data_sent);
2661 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_acked,
2662 &prev.tcps_tfo_syn_data_acked, &stat.tfo_syn_data_acked);
2663 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_loss,
2664 &prev.tcps_tfo_syn_loss, &stat.tfo_syn_loss);
2665 tcp_cumulative_stat(tcpstat.tcps_tfo_blackhole,
2666 &prev.tcps_tfo_blackhole, &stat.tfo_blackhole);
2667 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_wrong,
2668 &prev.tcps_tfo_cookie_wrong, &stat.tfo_cookie_wrong);
2669 tcp_cumulative_stat(tcpstat.tcps_tfo_no_cookie_rcv,
2670 &prev.tcps_tfo_no_cookie_rcv, &stat.tfo_no_cookie_rcv);
2671 tcp_cumulative_stat(tcpstat.tcps_tfo_heuristics_disable,
2672 &prev.tcps_tfo_heuristics_disable, &stat.tfo_heuristics_disable);
2673 tcp_cumulative_stat(tcpstat.tcps_tfo_sndblackhole,
2674 &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole);
2675
2676
2677 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt,
2678 &prev.tcps_mptcp_handover_attempt, &stat.mptcp_handover_attempt);
2679 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt,
2680 &prev.tcps_mptcp_interactive_attempt, &stat.mptcp_interactive_attempt);
2681 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt,
2682 &prev.tcps_mptcp_aggregate_attempt, &stat.mptcp_aggregate_attempt);
2683 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt,
2684 &prev.tcps_mptcp_fp_handover_attempt, &stat.mptcp_fp_handover_attempt);
2685 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt,
2686 &prev.tcps_mptcp_fp_interactive_attempt, &stat.mptcp_fp_interactive_attempt);
2687 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt,
2688 &prev.tcps_mptcp_fp_aggregate_attempt, &stat.mptcp_fp_aggregate_attempt);
2689 tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback,
2690 &prev.tcps_mptcp_heuristic_fallback, &stat.mptcp_heuristic_fallback);
2691 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback,
2692 &prev.tcps_mptcp_fp_heuristic_fallback, &stat.mptcp_fp_heuristic_fallback);
2693 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi,
2694 &prev.tcps_mptcp_handover_success_wifi, &stat.mptcp_handover_success_wifi);
2695 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell,
2696 &prev.tcps_mptcp_handover_success_cell, &stat.mptcp_handover_success_cell);
2697 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success,
2698 &prev.tcps_mptcp_interactive_success, &stat.mptcp_interactive_success);
2699 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success,
2700 &prev.tcps_mptcp_aggregate_success, &stat.mptcp_aggregate_success);
2701 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi,
2702 &prev.tcps_mptcp_fp_handover_success_wifi, &stat.mptcp_fp_handover_success_wifi);
2703 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell,
2704 &prev.tcps_mptcp_fp_handover_success_cell, &stat.mptcp_fp_handover_success_cell);
2705 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success,
2706 &prev.tcps_mptcp_fp_interactive_success, &stat.mptcp_fp_interactive_success);
2707 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success,
2708 &prev.tcps_mptcp_fp_aggregate_success, &stat.mptcp_fp_aggregate_success);
2709 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi,
2710 &prev.tcps_mptcp_handover_cell_from_wifi, &stat.mptcp_handover_cell_from_wifi);
2711 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell,
2712 &prev.tcps_mptcp_handover_wifi_from_cell, &stat.mptcp_handover_wifi_from_cell);
2713 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi,
2714 &prev.tcps_mptcp_interactive_cell_from_wifi, &stat.mptcp_interactive_cell_from_wifi);
2715 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes,
2716 &prev.tcps_mptcp_handover_cell_bytes, &stat.mptcp_handover_cell_bytes);
2717 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes,
2718 &prev.tcps_mptcp_interactive_cell_bytes, &stat.mptcp_interactive_cell_bytes);
2719 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes,
2720 &prev.tcps_mptcp_aggregate_cell_bytes, &stat.mptcp_aggregate_cell_bytes);
2721 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes,
2722 &prev.tcps_mptcp_handover_all_bytes, &stat.mptcp_handover_all_bytes);
2723 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes,
2724 &prev.tcps_mptcp_interactive_all_bytes, &stat.mptcp_interactive_all_bytes);
2725 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes,
2726 &prev.tcps_mptcp_aggregate_all_bytes, &stat.mptcp_aggregate_all_bytes);
2727 tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi,
2728 &prev.tcps_mptcp_back_to_wifi, &stat.mptcp_back_to_wifi);
2729 tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy,
2730 &prev.tcps_mptcp_wifi_proxy, &stat.mptcp_wifi_proxy);
2731 tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy,
2732 &prev.tcps_mptcp_cell_proxy, &stat.mptcp_cell_proxy);
2733 tcp_cumulative_stat(tcpstat.tcps_mptcp_triggered_cell,
2734 &prev.tcps_mptcp_triggered_cell, &stat.mptcp_triggered_cell);
2735
2736 nstat_sysinfo_send_data(&data);
2737
2738 #undef stat
2739 }
2740
2741 void
tcp_interface_send_probe(u_int16_t probe_if_index)2742 tcp_interface_send_probe(u_int16_t probe_if_index)
2743 {
2744 int32_t offset = 0;
2745 struct tcptimerlist *listp = &tcp_timer_list;
2746
2747 /* Make sure TCP clock is up to date */
2748 calculate_tcp_clock();
2749
2750 lck_mtx_lock(&listp->mtx);
2751 if (listp->probe_if_index > 0 && listp->probe_if_index != probe_if_index) {
2752 tcpstat.tcps_probe_if_conflict++;
2753 os_log(OS_LOG_DEFAULT,
2754 "%s: probe_if_index %u conflicts with %u, tcps_probe_if_conflict %u\n",
2755 __func__, probe_if_index, listp->probe_if_index,
2756 tcpstat.tcps_probe_if_conflict);
2757 goto done;
2758 }
2759
2760 listp->probe_if_index = probe_if_index;
2761 if (listp->running) {
2762 os_log(OS_LOG_DEFAULT, "%s: timer list already running for if_index %u\n",
2763 __func__, probe_if_index);
2764 goto done;
2765 }
2766
2767 /*
2768 * Reschedule the timerlist to run within the next 10ms, which is
2769 * the fastest that we can do.
2770 */
2771 offset = TCP_TIMER_10MS_QUANTUM;
2772 if (listp->scheduled) {
2773 int32_t diff;
2774 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2775 if (diff <= 0) {
2776 /* The timer will fire sooner than what's needed */
2777 os_log(OS_LOG_DEFAULT,
2778 "%s: timer will fire sooner than needed for if_index %u\n",
2779 __func__, probe_if_index);
2780 goto done;
2781 }
2782 }
2783 listp->mode = TCP_TIMERLIST_10MS_MODE;
2784 listp->idleruns = 0;
2785
2786 tcp_sched_timerlist(offset);
2787
2788 done:
2789 lck_mtx_unlock(&listp->mtx);
2790 return;
2791 }
2792
2793 /*
2794 * Enable read probes on this connection, if:
2795 * - it is in established state
2796 * - doesn't have any data outstanding
2797 * - the outgoing ifp matches
2798 * - we have not already sent any read probes
2799 */
2800 static void
tcp_enable_read_probe(struct tcpcb * tp,struct ifnet * ifp)2801 tcp_enable_read_probe(struct tcpcb *tp, struct ifnet *ifp)
2802 {
2803 if (tp->t_state == TCPS_ESTABLISHED &&
2804 tp->snd_max == tp->snd_una &&
2805 tp->t_inpcb->inp_last_outifp == ifp &&
2806 !(tp->t_flagsext & TF_DETECT_READSTALL) &&
2807 tp->t_rtimo_probes == 0) {
2808 tp->t_flagsext |= TF_DETECT_READSTALL;
2809 tp->t_rtimo_probes = 0;
2810 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
2811 TCP_TIMER_10MS_QUANTUM);
2812 if (tp->tentry.index == TCPT_NONE) {
2813 tp->tentry.index = TCPT_KEEP;
2814 tp->tentry.runtime = tcp_now +
2815 TCP_TIMER_10MS_QUANTUM;
2816 } else {
2817 int32_t diff = 0;
2818
2819 /* Reset runtime to be in next 10ms */
2820 diff = timer_diff(tp->tentry.runtime, 0,
2821 tcp_now, TCP_TIMER_10MS_QUANTUM);
2822 if (diff > 0) {
2823 tp->tentry.index = TCPT_KEEP;
2824 tp->tentry.runtime = tcp_now +
2825 TCP_TIMER_10MS_QUANTUM;
2826 if (tp->tentry.runtime == 0) {
2827 tp->tentry.runtime++;
2828 }
2829 }
2830 }
2831 }
2832 }
2833
2834 /*
2835 * Disable read probe and reset the keep alive timer
2836 */
2837 static void
tcp_disable_read_probe(struct tcpcb * tp)2838 tcp_disable_read_probe(struct tcpcb *tp)
2839 {
2840 if (tp->t_adaptive_rtimo == 0 &&
2841 ((tp->t_flagsext & TF_DETECT_READSTALL) ||
2842 tp->t_rtimo_probes > 0)) {
2843 tcp_keepalive_reset(tp);
2844
2845 if (tp->t_mpsub) {
2846 mptcp_reset_keepalive(tp);
2847 }
2848 }
2849 }
2850
2851 /*
2852 * Reschedule the tcp timerlist in the next 10ms to re-enable read/write
2853 * probes on connections going over a particular interface.
2854 */
2855 void
tcp_probe_connectivity(struct ifnet * ifp,u_int32_t enable)2856 tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable)
2857 {
2858 int32_t offset;
2859 struct tcptimerlist *listp = &tcp_timer_list;
2860 struct inpcbinfo *pcbinfo = &tcbinfo;
2861 struct inpcb *inp, *nxt;
2862
2863 if (ifp == NULL) {
2864 return;
2865 }
2866
2867 /* update clock */
2868 calculate_tcp_clock();
2869
2870 /*
2871 * Enable keep alive timer on all connections that are
2872 * active/established on this interface.
2873 */
2874 lck_rw_lock_shared(&pcbinfo->ipi_lock);
2875
2876 LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, nxt) {
2877 struct tcpcb *tp = NULL;
2878 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) ==
2879 WNT_STOPUSING) {
2880 continue;
2881 }
2882
2883 /* Acquire lock to look at the state of the connection */
2884 socket_lock(inp->inp_socket, 1);
2885
2886 /* Release the want count */
2887 if (inp->inp_ppcb == NULL ||
2888 (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) {
2889 socket_unlock(inp->inp_socket, 1);
2890 continue;
2891 }
2892 tp = intotcpcb(inp);
2893 if (enable) {
2894 tcp_enable_read_probe(tp, ifp);
2895 } else {
2896 tcp_disable_read_probe(tp);
2897 }
2898
2899 socket_unlock(inp->inp_socket, 1);
2900 }
2901 lck_rw_done(&pcbinfo->ipi_lock);
2902
2903 lck_mtx_lock(&listp->mtx);
2904 if (listp->running) {
2905 listp->pref_mode |= TCP_TIMERLIST_10MS_MODE;
2906 goto done;
2907 }
2908
2909 /* Reschedule within the next 10ms */
2910 offset = TCP_TIMER_10MS_QUANTUM;
2911 if (listp->scheduled) {
2912 int32_t diff;
2913 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2914 if (diff <= 0) {
2915 /* The timer will fire sooner than what's needed */
2916 goto done;
2917 }
2918 }
2919 listp->mode = TCP_TIMERLIST_10MS_MODE;
2920 listp->idleruns = 0;
2921
2922 tcp_sched_timerlist(offset);
2923 done:
2924 lck_mtx_unlock(&listp->mtx);
2925 return;
2926 }
2927
2928 inline void
tcp_update_mss_core(struct tcpcb * tp,struct ifnet * ifp)2929 tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp)
2930 {
2931 struct if_cellular_status_v1 *ifsr;
2932 u_int32_t optlen;
2933 ifsr = &ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
2934 if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) {
2935 optlen = tp->t_maxopd - tp->t_maxseg;
2936
2937 if (ifsr->mss_recommended ==
2938 IF_CELL_UL_MSS_RECOMMENDED_NONE &&
2939 tp->t_cached_maxopd > 0 &&
2940 tp->t_maxopd < tp->t_cached_maxopd) {
2941 tp->t_maxopd = tp->t_cached_maxopd;
2942 tcpstat.tcps_mss_to_default++;
2943 } else if (ifsr->mss_recommended ==
2944 IF_CELL_UL_MSS_RECOMMENDED_MEDIUM &&
2945 tp->t_maxopd > tcp_mss_rec_medium) {
2946 tp->t_cached_maxopd = tp->t_maxopd;
2947 tp->t_maxopd = tcp_mss_rec_medium;
2948 tcpstat.tcps_mss_to_medium++;
2949 } else if (ifsr->mss_recommended ==
2950 IF_CELL_UL_MSS_RECOMMENDED_LOW &&
2951 tp->t_maxopd > tcp_mss_rec_low) {
2952 tp->t_cached_maxopd = tp->t_maxopd;
2953 tp->t_maxopd = tcp_mss_rec_low;
2954 tcpstat.tcps_mss_to_low++;
2955 }
2956 tp->t_maxseg = tp->t_maxopd - optlen;
2957
2958 /*
2959 * clear the cached value if it is same as the current
2960 */
2961 if (tp->t_maxopd == tp->t_cached_maxopd) {
2962 tp->t_cached_maxopd = 0;
2963 }
2964 }
2965 }
2966
2967 void
tcp_update_mss_locked(struct socket * so,struct ifnet * ifp)2968 tcp_update_mss_locked(struct socket *so, struct ifnet *ifp)
2969 {
2970 struct inpcb *inp = sotoinpcb(so);
2971 struct tcpcb *tp = intotcpcb(inp);
2972
2973 if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) {
2974 return;
2975 }
2976
2977 if (!IFNET_IS_CELLULAR(ifp)) {
2978 /*
2979 * This optimization is implemented for cellular
2980 * networks only
2981 */
2982 return;
2983 }
2984 if (tp->t_state <= TCPS_CLOSE_WAIT) {
2985 /*
2986 * If the connection is currently doing or has done PMTU
2987 * blackhole detection, do not change the MSS
2988 */
2989 if (tp->t_flags & TF_BLACKHOLE) {
2990 return;
2991 }
2992 if (ifp->if_link_status == NULL) {
2993 return;
2994 }
2995 tcp_update_mss_core(tp, ifp);
2996 }
2997 }
2998
2999 void
tcp_itimer(struct inpcbinfo * ipi)3000 tcp_itimer(struct inpcbinfo *ipi)
3001 {
3002 struct inpcb *inp, *nxt;
3003
3004 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
3005 if (tcp_itimer_done == TRUE) {
3006 tcp_itimer_done = FALSE;
3007 os_atomic_inc(&ipi->ipi_timer_req.intimer_fast, relaxed);
3008 return;
3009 }
3010 /* Upgrade failed, lost lock now take it again exclusive */
3011 lck_rw_lock_exclusive(&ipi->ipi_lock);
3012 }
3013 tcp_itimer_done = TRUE;
3014
3015 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
3016 struct socket *so;
3017 struct ifnet *ifp;
3018
3019 if (inp->inp_ppcb == NULL ||
3020 in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) {
3021 continue;
3022 }
3023 so = inp->inp_socket;
3024 ifp = inp->inp_last_outifp;
3025 socket_lock(so, 1);
3026 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
3027 socket_unlock(so, 1);
3028 continue;
3029 }
3030 so_check_extended_bk_idle_time(so);
3031 if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) {
3032 tcp_update_mss_locked(so, NULL);
3033 }
3034 socket_unlock(so, 1);
3035
3036 /*
3037 * Defunct all system-initiated background sockets if the
3038 * socket is using the cellular interface and the interface
3039 * has its LQM set to abort.
3040 */
3041 if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) &&
3042 IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) &&
3043 ifp != NULL && IFNET_IS_CELLULAR(ifp) &&
3044 (ifp->if_interface_state.valid_bitmask &
3045 IF_INTERFACE_STATE_LQM_STATE_VALID) &&
3046 ifp->if_interface_state.lqm_state ==
3047 IFNET_LQM_THRESH_ABORT) {
3048 socket_defunct(current_proc(), so,
3049 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL);
3050 }
3051 }
3052
3053 ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT);
3054 lck_rw_done(&ipi->ipi_lock);
3055 }
3056
3057 void
tcp_set_link_heur_rtomin(struct tcpcb * tp,ifnet_t ifp)3058 tcp_set_link_heur_rtomin(struct tcpcb *tp, ifnet_t ifp)
3059 {
3060 if ((tcp_link_heuristics_flags & TCP_LINK_HEUR_RTOMIN) != 0 &&
3061 ifp != NULL && if_link_heuristics_enabled(ifp)) {
3062 if (tp->t_rxtcur < tcp_link_heuristics_rto_min) {
3063 IF_TCP_STATINC(ifp, linkheur_rxmtfloor);
3064 tp->t_rxtcur = tcp_link_heuristics_rto_min;
3065 }
3066 }
3067 }
3068