1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * Copyright (c) 1982, 1986, 1988, 1990, 1993, 1995
30 * The Regents of the University of California. All rights reserved.
31 *
32 * Redistribution and use in source and binary forms, with or without
33 * modification, are permitted provided that the following conditions
34 * are met:
35 * 1. Redistributions of source code must retain the above copyright
36 * notice, this list of conditions and the following disclaimer.
37 * 2. Redistributions in binary form must reproduce the above copyright
38 * notice, this list of conditions and the following disclaimer in the
39 * documentation and/or other materials provided with the distribution.
40 * 3. All advertising materials mentioning features or use of this software
41 * must display the following acknowledgement:
42 * This product includes software developed by the University of
43 * California, Berkeley and its contributors.
44 * 4. Neither the name of the University nor the names of its contributors
45 * may be used to endorse or promote products derived from this software
46 * without specific prior written permission.
47 *
48 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58 * SUCH DAMAGE.
59 *
60 * @(#)tcp_timer.c 8.2 (Berkeley) 5/24/95
61 * $FreeBSD: src/sys/netinet/tcp_timer.c,v 1.34.2.11 2001/08/22 00:59:12 silby Exp $
62 */
63
64 #include "tcp_includes.h"
65
66 #include <sys/param.h>
67 #include <sys/systm.h>
68 #include <sys/kernel.h>
69 #include <sys/mbuf.h>
70 #include <sys/sysctl.h>
71 #include <sys/socket.h>
72 #include <sys/socketvar.h>
73 #include <sys/protosw.h>
74 #include <sys/domain.h>
75 #include <sys/mcache.h>
76 #include <sys/queue.h>
77 #include <kern/locks.h>
78 #include <kern/cpu_number.h> /* before tcp_seq.h, for tcp_random18() */
79 #include <mach/boolean.h>
80
81 #include <net/route.h>
82 #include <net/if_var.h>
83 #include <net/ntstat.h>
84
85 #include <netinet/in.h>
86 #include <netinet/in_systm.h>
87 #include <netinet/in_pcb.h>
88 #include <netinet/in_var.h>
89 #include <netinet6/in6_pcb.h>
90 #include <netinet/ip_var.h>
91 #include <netinet/tcp.h>
92 #include <netinet/tcp_cache.h>
93 #include <netinet/tcp_fsm.h>
94 #include <netinet/tcp_seq.h>
95 #include <netinet/tcp_timer.h>
96 #include <netinet/tcp_var.h>
97 #include <netinet/tcp_cc.h>
98 #include <netinet6/tcp6_var.h>
99 #include <netinet/tcpip.h>
100 #if TCPDEBUG
101 #include <netinet/tcp_debug.h>
102 #endif
103 #include <netinet/tcp_log.h>
104
105 #include <sys/kdebug.h>
106 #include <mach/sdt.h>
107 #include <netinet/mptcp_var.h>
108
109 /* Max number of times a stretch ack can be delayed on a connection */
110 #define TCP_STRETCHACK_DELAY_THRESHOLD 5
111
112 /*
113 * If the host processor has been sleeping for too long, this is the threshold
114 * used to avoid sending stale retransmissions.
115 */
116 #define TCP_SLEEP_TOO_LONG (10 * 60 * 1000) /* 10 minutes in ms */
117
118 /* tcp timer list */
119 struct tcptimerlist tcp_timer_list;
120
121 /* List of pcbs in timewait state, protected by tcbinfo's ipi_lock */
122 struct tcptailq tcp_tw_tailq;
123
124
125 static int
126 sysctl_msec_to_ticks SYSCTL_HANDLER_ARGS
127 {
128 #pragma unused(arg2)
129 int error, temp;
130 long s, tt;
131
132 tt = *(int *)arg1;
133 s = tt * 1000 / TCP_RETRANSHZ;
134 if (tt < 0 || s > INT_MAX) {
135 return EINVAL;
136 }
137 temp = (int)s;
138
139 error = sysctl_handle_int(oidp, &temp, 0, req);
140 if (error || !req->newptr) {
141 return error;
142 }
143
144 tt = (long)temp * TCP_RETRANSHZ / 1000;
145 if (tt < 1 || tt > INT_MAX) {
146 return EINVAL;
147 }
148
149 *(int *)arg1 = (int)tt;
150 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, *(int*)arg1);
151 return 0;
152 }
153
154 #if SYSCTL_SKMEM
155 int tcp_keepinit = TCPTV_KEEP_INIT;
156 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
157 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
158 &tcp_keepinit, offsetof(skmem_sysctl, tcp.keepinit),
159 sysctl_msec_to_ticks, "I", "");
160
161 int tcp_keepidle = TCPTV_KEEP_IDLE;
162 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
163 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
164 &tcp_keepidle, offsetof(skmem_sysctl, tcp.keepidle),
165 sysctl_msec_to_ticks, "I", "");
166
167 int tcp_keepintvl = TCPTV_KEEPINTVL;
168 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
169 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
170 &tcp_keepintvl, offsetof(skmem_sysctl, tcp.keepintvl),
171 sysctl_msec_to_ticks, "I", "");
172
173 SYSCTL_SKMEM_TCP_INT(OID_AUTO, keepcnt,
174 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
175 int, tcp_keepcnt, TCPTV_KEEPCNT, "number of times to repeat keepalive");
176
177 int tcp_msl = TCPTV_MSL;
178 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
179 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
180 &tcp_msl, offsetof(skmem_sysctl, tcp.msl),
181 sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
182 #else /* SYSCTL_SKMEM */
183 int tcp_keepinit;
184 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINIT, keepinit,
185 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
186 &tcp_keepinit, 0, sysctl_msec_to_ticks, "I", "");
187
188 int tcp_keepidle;
189 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPIDLE, keepidle,
190 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
191 &tcp_keepidle, 0, sysctl_msec_to_ticks, "I", "");
192
193 int tcp_keepintvl;
194 SYSCTL_PROC(_net_inet_tcp, TCPCTL_KEEPINTVL, keepintvl,
195 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
196 &tcp_keepintvl, 0, sysctl_msec_to_ticks, "I", "");
197
198 int tcp_keepcnt;
199 SYSCTL_INT(_net_inet_tcp, OID_AUTO, keepcnt,
200 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
201 &tcp_keepcnt, 0, "number of times to repeat keepalive");
202
203 int tcp_msl;
204 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, msl,
205 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
206 &tcp_msl, 0, sysctl_msec_to_ticks, "I", "Maximum segment lifetime");
207 #endif /* SYSCTL_SKMEM */
208
209 /*
210 * Avoid DoS with connections half-closed in TIME_WAIT_2
211 */
212 int tcp_fin_timeout = TCPTV_FINWAIT2;
213
214 static int
215 sysctl_tcp_fin_timeout SYSCTL_HANDLER_ARGS
216 {
217 #pragma unused(arg2)
218 int error;
219 int value = tcp_fin_timeout;
220
221 error = sysctl_handle_int(oidp, &value, 0, req);
222 if (error != 0 || req->newptr == USER_ADDR_NULL) {
223 return error;
224 }
225
226 if (value == -1) {
227 /* Reset to default value */
228 value = TCPTV_FINWAIT2;
229 } else {
230 /* Convert from milliseconds */
231 long big_value = value * TCP_RETRANSHZ / 1000;
232
233 if (big_value < 0 || big_value > INT_MAX) {
234 return EINVAL;
235 }
236 value = (int)big_value;
237 }
238 tcp_fin_timeout = value;
239 SYSCTL_SKMEM_UPDATE_AT_OFFSET(arg2, value);
240 return 0;
241 }
242
243 #if SYSCTL_SKMEM
244 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
245 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
246 &tcp_fin_timeout, offsetof(skmem_sysctl, tcp.fin_timeout),
247 sysctl_tcp_fin_timeout, "I", "");
248 #else /* SYSCTL_SKMEM */
249 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, fin_timeout,
250 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
251 &tcp_fin_timeout, 0,
252 sysctl_tcp_fin_timeout, "I", "");
253 #endif /* SYSCTL_SKMEM */
254
255 /*
256 * Avoid DoS via TCP Robustness in Persist Condition
257 * (see http://www.ietf.org/id/draft-ananth-tcpm-persist-02.txt)
258 * by allowing a system wide maximum persistence timeout value when in
259 * Zero Window Probe mode.
260 *
261 * Expressed in milliseconds to be consistent without timeout related
262 * values, the TCP socket option is in seconds.
263 */
264 #if SYSCTL_SKMEM
265 u_int32_t tcp_max_persist_timeout = 0;
266 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
267 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
268 &tcp_max_persist_timeout, offsetof(skmem_sysctl, tcp.max_persist_timeout),
269 sysctl_msec_to_ticks, "I", "Maximum persistence timeout for ZWP");
270 #else /* SYSCTL_SKMEM */
271 u_int32_t tcp_max_persist_timeout = 0;
272 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, max_persist_timeout,
273 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
274 &tcp_max_persist_timeout, 0, sysctl_msec_to_ticks, "I",
275 "Maximum persistence timeout for ZWP");
276 #endif /* SYSCTL_SKMEM */
277
278 SYSCTL_SKMEM_TCP_INT(OID_AUTO, always_keepalive,
279 CTLFLAG_RW | CTLFLAG_LOCKED, static int, always_keepalive, 0,
280 "Assume SO_KEEPALIVE on all TCP connections");
281
282 /*
283 * This parameter determines how long the timer list will stay in fast or
284 * quick mode even though all connections are idle. In this state, the
285 * timer will run more frequently anticipating new data.
286 */
287 SYSCTL_SKMEM_TCP_INT(OID_AUTO, timer_fastmode_idlemax,
288 CTLFLAG_RW | CTLFLAG_LOCKED, int, timer_fastmode_idlemax,
289 TCP_FASTMODE_IDLERUN_MAX, "Maximum idle generations in fast mode");
290
291 /*
292 * See tcp_syn_backoff[] for interval values between SYN retransmits;
293 * the value set below defines the number of retransmits, before we
294 * disable the timestamp and window scaling options during subsequent
295 * SYN retransmits. Setting it to 0 disables the dropping off of those
296 * two options.
297 */
298 SYSCTL_SKMEM_TCP_INT(OID_AUTO, broken_peer_syn_rexmit_thres,
299 CTLFLAG_RW | CTLFLAG_LOCKED, static int, tcp_broken_peer_syn_rxmit_thres,
300 10, "Number of retransmitted SYNs before disabling RFC 1323 "
301 "options on local connections");
302
303 static int tcp_timer_advanced = 0;
304 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_timer_advanced,
305 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_timer_advanced, 0,
306 "Number of times one of the timers was advanced");
307
308 static int tcp_resched_timerlist = 0;
309 SYSCTL_INT(_net_inet_tcp, OID_AUTO, tcp_resched_timerlist,
310 CTLFLAG_RD | CTLFLAG_LOCKED, &tcp_resched_timerlist, 0,
311 "Number of times timer list was rescheduled as part of processing a packet");
312
313 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_detection,
314 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_detect, 1,
315 "Path MTU Discovery Black Hole Detection");
316
317 SYSCTL_SKMEM_TCP_INT(OID_AUTO, pmtud_blackhole_mss,
318 CTLFLAG_RW | CTLFLAG_LOCKED, int, tcp_pmtud_black_hole_mss, 1200,
319 "Path MTU Discovery Black Hole Detection lowered MSS");
320
321 #if (DEBUG || DEVELOPMENT)
322 int tcp_probe_if_fix_port = 0;
323 SYSCTL_INT(_net_inet_tcp, OID_AUTO, probe_if_fix_port,
324 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
325 &tcp_probe_if_fix_port, 0, "");
326 #endif /* (DEBUG || DEVELOPMENT) */
327
328 static u_int32_t tcp_mss_rec_medium = 1200;
329 static u_int32_t tcp_mss_rec_low = 512;
330
331 #define TCP_REPORT_STATS_INTERVAL 43200 /* 12 hours, in seconds */
332 int tcp_report_stats_interval = TCP_REPORT_STATS_INTERVAL;
333
334 /* performed garbage collection of "used" sockets */
335 static boolean_t tcp_gc_done = FALSE;
336
337 /* max idle probes */
338 int tcp_maxpersistidle = TCPTV_KEEP_IDLE;
339
340 /*
341 * TCP delack timer is set to 100 ms. Since the processing of timer list
342 * in fast mode will happen no faster than 100 ms, the delayed ack timer
343 * will fire some where between 100 and 200 ms.
344 */
345 int tcp_delack = TCP_RETRANSHZ / 10;
346
347 #if MPTCP
348 /*
349 * MP_JOIN retransmission of 3rd ACK will be every 500 msecs without backoff
350 */
351 int tcp_jack_rxmt = TCP_RETRANSHZ / 2;
352 #endif /* MPTCP */
353
354 static boolean_t tcp_itimer_done = FALSE;
355
356 static void tcp_remove_timer(struct tcpcb *tp);
357 static void tcp_sched_timerlist(uint32_t offset);
358 static u_int32_t tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *mode,
359 u_int16_t probe_if_index);
360 static inline void tcp_set_lotimer_index(struct tcpcb *);
361 __private_extern__ void tcp_remove_from_time_wait(struct inpcb *inp);
362 static inline void tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp);
363 __private_extern__ void tcp_report_stats(void);
364
365 static u_int64_t tcp_last_report_time;
366
367 /*
368 * Structure to store previously reported stats so that we can send
369 * incremental changes in each report interval.
370 */
371 struct tcp_last_report_stats {
372 u_int32_t tcps_connattempt;
373 u_int32_t tcps_accepts;
374 u_int32_t tcps_ecn_client_setup;
375 u_int32_t tcps_ecn_server_setup;
376 u_int32_t tcps_ecn_client_success;
377 u_int32_t tcps_ecn_server_success;
378 u_int32_t tcps_ecn_not_supported;
379 u_int32_t tcps_ecn_lost_syn;
380 u_int32_t tcps_ecn_lost_synack;
381 u_int32_t tcps_ecn_recv_ce;
382 u_int32_t tcps_ecn_recv_ece;
383 u_int32_t tcps_ecn_sent_ece;
384 u_int32_t tcps_ecn_conn_recv_ce;
385 u_int32_t tcps_ecn_conn_recv_ece;
386 u_int32_t tcps_ecn_conn_plnoce;
387 u_int32_t tcps_ecn_conn_pl_ce;
388 u_int32_t tcps_ecn_conn_nopl_ce;
389 u_int32_t tcps_ecn_fallback_synloss;
390 u_int32_t tcps_ecn_fallback_reorder;
391 u_int32_t tcps_ecn_fallback_ce;
392
393 /* TFO-related statistics */
394 u_int32_t tcps_tfo_syn_data_rcv;
395 u_int32_t tcps_tfo_cookie_req_rcv;
396 u_int32_t tcps_tfo_cookie_sent;
397 u_int32_t tcps_tfo_cookie_invalid;
398 u_int32_t tcps_tfo_cookie_req;
399 u_int32_t tcps_tfo_cookie_rcv;
400 u_int32_t tcps_tfo_syn_data_sent;
401 u_int32_t tcps_tfo_syn_data_acked;
402 u_int32_t tcps_tfo_syn_loss;
403 u_int32_t tcps_tfo_blackhole;
404 u_int32_t tcps_tfo_cookie_wrong;
405 u_int32_t tcps_tfo_no_cookie_rcv;
406 u_int32_t tcps_tfo_heuristics_disable;
407 u_int32_t tcps_tfo_sndblackhole;
408
409 /* MPTCP-related statistics */
410 u_int32_t tcps_mptcp_handover_attempt;
411 u_int32_t tcps_mptcp_interactive_attempt;
412 u_int32_t tcps_mptcp_aggregate_attempt;
413 u_int32_t tcps_mptcp_fp_handover_attempt;
414 u_int32_t tcps_mptcp_fp_interactive_attempt;
415 u_int32_t tcps_mptcp_fp_aggregate_attempt;
416 u_int32_t tcps_mptcp_heuristic_fallback;
417 u_int32_t tcps_mptcp_fp_heuristic_fallback;
418 u_int32_t tcps_mptcp_handover_success_wifi;
419 u_int32_t tcps_mptcp_handover_success_cell;
420 u_int32_t tcps_mptcp_interactive_success;
421 u_int32_t tcps_mptcp_aggregate_success;
422 u_int32_t tcps_mptcp_fp_handover_success_wifi;
423 u_int32_t tcps_mptcp_fp_handover_success_cell;
424 u_int32_t tcps_mptcp_fp_interactive_success;
425 u_int32_t tcps_mptcp_fp_aggregate_success;
426 u_int32_t tcps_mptcp_handover_cell_from_wifi;
427 u_int32_t tcps_mptcp_handover_wifi_from_cell;
428 u_int32_t tcps_mptcp_interactive_cell_from_wifi;
429 u_int64_t tcps_mptcp_handover_cell_bytes;
430 u_int64_t tcps_mptcp_interactive_cell_bytes;
431 u_int64_t tcps_mptcp_aggregate_cell_bytes;
432 u_int64_t tcps_mptcp_handover_all_bytes;
433 u_int64_t tcps_mptcp_interactive_all_bytes;
434 u_int64_t tcps_mptcp_aggregate_all_bytes;
435 u_int32_t tcps_mptcp_back_to_wifi;
436 u_int32_t tcps_mptcp_wifi_proxy;
437 u_int32_t tcps_mptcp_cell_proxy;
438 u_int32_t tcps_mptcp_triggered_cell;
439 };
440
441
442 /* Returns true if the timer is on the timer list */
443 #define TIMER_IS_ON_LIST(tp) ((tp)->t_flags & TF_TIMER_ONLIST)
444
445 /* Run the TCP timerlist atleast once every hour */
446 #define TCP_TIMERLIST_MAX_OFFSET (60 * 60 * TCP_RETRANSHZ)
447
448
449 static void add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay);
450 static boolean_t tcp_garbage_collect(struct inpcb *, int);
451
452 #define TIMERENTRY_TO_TP(te) ((struct tcpcb *)((uintptr_t)te - offsetof(struct tcpcb, tentry.le.le_next)))
453
454 #define VERIFY_NEXT_LINK(elm, field) do { \
455 if (LIST_NEXT((elm),field) != NULL && \
456 LIST_NEXT((elm),field)->field.le_prev != \
457 &((elm)->field.le_next)) \
458 panic("Bad link elm %p next->prev != elm", (elm)); \
459 } while(0)
460
461 #define VERIFY_PREV_LINK(elm, field) do { \
462 if (*(elm)->field.le_prev != (elm)) \
463 panic("Bad link elm %p prev->next != elm", (elm)); \
464 } while(0)
465
466 #define TCP_SET_TIMER_MODE(mode, i) do { \
467 if (IS_TIMER_HZ_10MS(i)) \
468 (mode) |= TCP_TIMERLIST_10MS_MODE; \
469 else if (IS_TIMER_HZ_100MS(i)) \
470 (mode) |= TCP_TIMERLIST_100MS_MODE; \
471 else \
472 (mode) |= TCP_TIMERLIST_500MS_MODE; \
473 } while(0)
474
475 #if (DEVELOPMENT || DEBUG)
476 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_medium,
477 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_medium, 0,
478 "Medium MSS based on recommendation in link status report");
479 SYSCTL_UINT(_net_inet_tcp, OID_AUTO, mss_rec_low,
480 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_mss_rec_low, 0,
481 "Low MSS based on recommendation in link status report");
482
483 static int32_t tcp_change_mss_recommended = 0;
484 static int
485 sysctl_change_mss_recommended SYSCTL_HANDLER_ARGS
486 {
487 #pragma unused(oidp, arg1, arg2)
488 int i, err = 0, changed = 0;
489 struct ifnet *ifp;
490 struct if_link_status ifsr;
491 struct if_cellular_status_v1 *new_cell_sr;
492 err = sysctl_io_number(req, tcp_change_mss_recommended,
493 sizeof(int32_t), &i, &changed);
494 if (changed) {
495 if (i < 0 || i > UINT16_MAX) {
496 return EINVAL;
497 }
498 ifnet_head_lock_shared();
499 TAILQ_FOREACH(ifp, &ifnet_head, if_link) {
500 if (IFNET_IS_CELLULAR(ifp)) {
501 bzero(&ifsr, sizeof(ifsr));
502 new_cell_sr = &ifsr.ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
503 ifsr.ifsr_version = IF_CELLULAR_STATUS_REPORT_CURRENT_VERSION;
504 ifsr.ifsr_len = sizeof(*new_cell_sr);
505
506 /* Set MSS recommended */
507 new_cell_sr->valid_bitmask |= IF_CELL_UL_MSS_RECOMMENDED_VALID;
508 new_cell_sr->mss_recommended = (uint16_t)i;
509 err = ifnet_link_status_report(ifp, new_cell_sr, sizeof(new_cell_sr));
510 if (err == 0) {
511 tcp_change_mss_recommended = i;
512 } else {
513 break;
514 }
515 }
516 }
517 ifnet_head_done();
518 }
519 return err;
520 }
521
522 SYSCTL_PROC(_net_inet_tcp, OID_AUTO, change_mss_recommended,
523 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_change_mss_recommended,
524 0, sysctl_change_mss_recommended, "IU", "Change MSS recommended");
525
526 SYSCTL_INT(_net_inet_tcp, OID_AUTO, report_stats_interval,
527 CTLFLAG_RW | CTLFLAG_LOCKED, &tcp_report_stats_interval, 0,
528 "Report stats interval");
529 #endif /* (DEVELOPMENT || DEBUG) */
530
531 /*
532 * Macro to compare two timers. If there is a reset of the sign bit,
533 * it is safe to assume that the timer has wrapped around. By doing
534 * signed comparision, we take care of wrap around such that the value
535 * with the sign bit reset is actually ahead of the other.
536 */
537 inline int32_t
timer_diff(uint32_t t1,uint32_t toff1,uint32_t t2,uint32_t toff2)538 timer_diff(uint32_t t1, uint32_t toff1, uint32_t t2, uint32_t toff2)
539 {
540 return (int32_t)((t1 + toff1) - (t2 + toff2));
541 }
542
543 /*
544 * Add to tcp timewait list, delay is given in milliseconds.
545 */
546 static void
add_to_time_wait_locked(struct tcpcb * tp,uint32_t delay)547 add_to_time_wait_locked(struct tcpcb *tp, uint32_t delay)
548 {
549 struct inpcbinfo *pcbinfo = &tcbinfo;
550 struct inpcb *inp = tp->t_inpcb;
551 uint32_t timer;
552
553 /* pcb list should be locked when we get here */
554 LCK_RW_ASSERT(&pcbinfo->ipi_lock, LCK_RW_ASSERT_EXCLUSIVE);
555
556 /* We may get here multiple times, so check */
557 if (!(inp->inp_flags2 & INP2_TIMEWAIT)) {
558 pcbinfo->ipi_twcount++;
559 inp->inp_flags2 |= INP2_TIMEWAIT;
560
561 /* Remove from global inp list */
562 LIST_REMOVE(inp, inp_list);
563 } else {
564 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
565 }
566
567 /* Compute the time at which this socket can be closed */
568 timer = tcp_now + delay;
569
570 /* We will use the TCPT_2MSL timer for tracking this delay */
571
572 if (TIMER_IS_ON_LIST(tp)) {
573 tcp_remove_timer(tp);
574 }
575 tp->t_timer[TCPT_2MSL] = timer;
576
577 TAILQ_INSERT_TAIL(&tcp_tw_tailq, tp, t_twentry);
578 }
579
580 void
add_to_time_wait(struct tcpcb * tp,uint32_t delay)581 add_to_time_wait(struct tcpcb *tp, uint32_t delay)
582 {
583 struct inpcbinfo *pcbinfo = &tcbinfo;
584 if (tp->t_inpcb->inp_socket->so_options & SO_NOWAKEFROMSLEEP) {
585 socket_post_kev_msg_closed(tp->t_inpcb->inp_socket);
586 }
587
588 tcp_del_fsw_flow(tp);
589
590 /* 19182803: Notify nstat that connection is closing before waiting. */
591 nstat_pcb_detach(tp->t_inpcb);
592
593 if (!lck_rw_try_lock_exclusive(&pcbinfo->ipi_lock)) {
594 socket_unlock(tp->t_inpcb->inp_socket, 0);
595 lck_rw_lock_exclusive(&pcbinfo->ipi_lock);
596 socket_lock(tp->t_inpcb->inp_socket, 0);
597 }
598 add_to_time_wait_locked(tp, delay);
599 lck_rw_done(&pcbinfo->ipi_lock);
600
601 inpcb_gc_sched(pcbinfo, INPCB_TIMER_LAZY);
602 }
603
604 /* If this is on time wait queue, remove it. */
605 void
tcp_remove_from_time_wait(struct inpcb * inp)606 tcp_remove_from_time_wait(struct inpcb *inp)
607 {
608 struct tcpcb *tp = intotcpcb(inp);
609 if (inp->inp_flags2 & INP2_TIMEWAIT) {
610 TAILQ_REMOVE(&tcp_tw_tailq, tp, t_twentry);
611 }
612 }
613
614 static boolean_t
tcp_garbage_collect(struct inpcb * inp,int istimewait)615 tcp_garbage_collect(struct inpcb *inp, int istimewait)
616 {
617 boolean_t active = FALSE;
618 struct socket *so, *mp_so = NULL;
619 struct tcpcb *tp;
620
621 so = inp->inp_socket;
622 tp = intotcpcb(inp);
623
624 if (so->so_flags & SOF_MP_SUBFLOW) {
625 mp_so = mptetoso(tptomptp(tp)->mpt_mpte);
626 if (!socket_try_lock(mp_so)) {
627 mp_so = NULL;
628 active = TRUE;
629 goto out;
630 }
631 if (mpsotomppcb(mp_so)->mpp_inside > 0) {
632 os_log(mptcp_log_handle, "%s - %lx: Still inside %d usecount %d\n", __func__,
633 (unsigned long)VM_KERNEL_ADDRPERM(mpsotompte(mp_so)),
634 mpsotomppcb(mp_so)->mpp_inside,
635 mp_so->so_usecount);
636 socket_unlock(mp_so, 0);
637 mp_so = NULL;
638 active = TRUE;
639 goto out;
640 }
641 /* We call socket_unlock with refcount further below */
642 mp_so->so_usecount++;
643 tptomptp(tp)->mpt_mpte->mpte_mppcb->mpp_inside++;
644 }
645
646 /*
647 * Skip if still in use or busy; it would have been more efficient
648 * if we were to test so_usecount against 0, but this isn't possible
649 * due to the current implementation of tcp_dropdropablreq() where
650 * overflow sockets that are eligible for garbage collection have
651 * their usecounts set to 1.
652 */
653 if (!lck_mtx_try_lock_spin(&inp->inpcb_mtx)) {
654 active = TRUE;
655 goto out;
656 }
657
658 /* Check again under the lock */
659 if (so->so_usecount > 1) {
660 if (inp->inp_wantcnt == WNT_STOPUSING) {
661 active = TRUE;
662 }
663 lck_mtx_unlock(&inp->inpcb_mtx);
664 goto out;
665 }
666
667 if (istimewait && TSTMP_GEQ(tcp_now, tp->t_timer[TCPT_2MSL]) &&
668 tp->t_state != TCPS_CLOSED) {
669 /* Become a regular mutex */
670 lck_mtx_convert_spin(&inp->inpcb_mtx);
671 tcp_close(tp);
672 }
673
674 /*
675 * Overflowed socket dropped from the listening queue? Do this
676 * only if we are called to clean up the time wait slots, since
677 * tcp_dropdropablreq() considers a socket to have been fully
678 * dropped after add_to_time_wait() is finished.
679 * Also handle the case of connections getting closed by the peer
680 * while in the queue as seen with rdar://6422317
681 *
682 */
683 if (so->so_usecount == 1 &&
684 ((istimewait && (so->so_flags & SOF_OVERFLOW)) ||
685 ((tp != NULL) && (tp->t_state == TCPS_CLOSED) &&
686 (so->so_head != NULL) &&
687 ((so->so_state & (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE)) ==
688 (SS_INCOMP | SS_CANTSENDMORE | SS_CANTRCVMORE))))) {
689 if (inp->inp_state != INPCB_STATE_DEAD) {
690 /* Become a regular mutex */
691 lck_mtx_convert_spin(&inp->inpcb_mtx);
692 if (SOCK_CHECK_DOM(so, PF_INET6)) {
693 in6_pcbdetach(inp);
694 } else {
695 in_pcbdetach(inp);
696 }
697 }
698 VERIFY(so->so_usecount > 0);
699 so->so_usecount--;
700 if (inp->inp_wantcnt == WNT_STOPUSING) {
701 active = TRUE;
702 }
703 lck_mtx_unlock(&inp->inpcb_mtx);
704 goto out;
705 } else if (inp->inp_wantcnt != WNT_STOPUSING) {
706 lck_mtx_unlock(&inp->inpcb_mtx);
707 active = FALSE;
708 goto out;
709 }
710
711 /*
712 * We get here because the PCB is no longer searchable
713 * (WNT_STOPUSING); detach (if needed) and dispose if it is dead
714 * (usecount is 0). This covers all cases, including overflow
715 * sockets and those that are considered as "embryonic",
716 * i.e. created by sonewconn() in TCP input path, and have
717 * not yet been committed. For the former, we reduce the usecount
718 * to 0 as done by the code above. For the latter, the usecount
719 * would have reduced to 0 as part calling soabort() when the
720 * socket is dropped at the end of tcp_input().
721 */
722 if (so->so_usecount == 0) {
723 DTRACE_TCP4(state__change, void, NULL, struct inpcb *, inp,
724 struct tcpcb *, tp, int32_t, TCPS_CLOSED);
725 /* Become a regular mutex */
726 lck_mtx_convert_spin(&inp->inpcb_mtx);
727
728 /*
729 * If this tp still happens to be on the timer list,
730 * take it out
731 */
732 if (TIMER_IS_ON_LIST(tp)) {
733 tcp_remove_timer(tp);
734 }
735
736 if (inp->inp_state != INPCB_STATE_DEAD) {
737 if (SOCK_CHECK_DOM(so, PF_INET6)) {
738 in6_pcbdetach(inp);
739 } else {
740 in_pcbdetach(inp);
741 }
742 }
743
744 if (mp_so) {
745 mptcp_subflow_del(tptomptp(tp)->mpt_mpte, tp->t_mpsub);
746
747 /* so is now unlinked from mp_so - let's drop the lock */
748 socket_unlock(mp_so, 1);
749 mp_so = NULL;
750 }
751
752 in_pcbdispose(inp);
753 active = FALSE;
754 goto out;
755 }
756
757 lck_mtx_unlock(&inp->inpcb_mtx);
758 active = TRUE;
759
760 out:
761 if (mp_so) {
762 socket_unlock(mp_so, 1);
763 }
764
765 return active;
766 }
767
768 /*
769 * TCP garbage collector callback (inpcb_timer_func_t).
770 *
771 * Returns the number of pcbs that will need to be gc-ed soon,
772 * returnining > 0 will keep timer active.
773 */
774 void
tcp_gc(struct inpcbinfo * ipi)775 tcp_gc(struct inpcbinfo *ipi)
776 {
777 struct inpcb *inp, *nxt;
778 struct tcpcb *tw_tp, *tw_ntp;
779 #if TCPDEBUG
780 int ostate;
781 #endif
782 #if KDEBUG
783 static int tws_checked = 0;
784 #endif
785
786 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_START, 0, 0, 0, 0, 0);
787
788 /*
789 * Update tcp_now here as it may get used while
790 * processing the slow timer.
791 */
792 calculate_tcp_clock();
793
794 /*
795 * Garbage collect socket/tcpcb: We need to acquire the list lock
796 * exclusively to do this
797 */
798
799 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
800 /* don't sweat it this time; cleanup was done last time */
801 if (tcp_gc_done == TRUE) {
802 tcp_gc_done = FALSE;
803 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END,
804 tws_checked, cur_tw_slot, 0, 0, 0);
805 /* Lock upgrade failed, give up this round */
806 atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1);
807 return;
808 }
809 /* Upgrade failed, lost lock now take it again exclusive */
810 lck_rw_lock_exclusive(&ipi->ipi_lock);
811 }
812 tcp_gc_done = TRUE;
813
814 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
815 if (tcp_garbage_collect(inp, 0)) {
816 atomic_add_32(&ipi->ipi_gc_req.intimer_fast, 1);
817 }
818 }
819
820 /* Now cleanup the time wait ones */
821 TAILQ_FOREACH_SAFE(tw_tp, &tcp_tw_tailq, t_twentry, tw_ntp) {
822 /*
823 * We check the timestamp here without holding the
824 * socket lock for better performance. If there are
825 * any pcbs in time-wait, the timer will get rescheduled.
826 * Hence some error in this check can be tolerated.
827 *
828 * Sometimes a socket on time-wait queue can be closed if
829 * 2MSL timer expired but the application still has a
830 * usecount on it.
831 */
832 if (tw_tp->t_state == TCPS_CLOSED ||
833 TSTMP_GEQ(tcp_now, tw_tp->t_timer[TCPT_2MSL])) {
834 if (tcp_garbage_collect(tw_tp->t_inpcb, 1)) {
835 atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1);
836 }
837 }
838 }
839
840 /* take into account pcbs that are still in time_wait_slots */
841 atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, ipi->ipi_twcount);
842
843 lck_rw_done(&ipi->ipi_lock);
844
845 /* Clean up the socache while we are here */
846 if (so_cache_timer()) {
847 atomic_add_32(&ipi->ipi_gc_req.intimer_lazy, 1);
848 }
849
850 KERNEL_DEBUG(DBG_FNC_TCP_SLOW | DBG_FUNC_END, tws_checked,
851 cur_tw_slot, 0, 0, 0);
852
853 return;
854 }
855
856 /*
857 * Cancel all timers for TCP tp.
858 */
859 void
tcp_canceltimers(struct tcpcb * tp)860 tcp_canceltimers(struct tcpcb *tp)
861 {
862 int i;
863
864 tcp_remove_timer(tp);
865 for (i = 0; i < TCPT_NTIMERS; i++) {
866 tp->t_timer[i] = 0;
867 }
868 tp->tentry.timer_start = tcp_now;
869 tp->tentry.index = TCPT_NONE;
870 }
871
872 int tcp_syn_backoff[TCP_MAXRXTSHIFT + 1] =
873 { 1, 1, 1, 1, 1, 2, 4, 8, 16, 32, 64, 64, 64 };
874
875 int tcp_backoff[TCP_MAXRXTSHIFT + 1] =
876 { 1, 2, 4, 8, 16, 32, 64, 64, 64, 64, 64, 64, 64 };
877
878 static int tcp_totbackoff = 511; /* sum of tcp_backoff[] */
879
880 void
tcp_rexmt_save_state(struct tcpcb * tp)881 tcp_rexmt_save_state(struct tcpcb *tp)
882 {
883 u_int32_t fsize;
884 if (TSTMP_SUPPORTED(tp)) {
885 /*
886 * Since timestamps are supported on the connection,
887 * we can do recovery as described in rfc 4015.
888 */
889 fsize = tp->snd_max - tp->snd_una;
890 tp->snd_ssthresh_prev = max(fsize, tp->snd_ssthresh);
891 tp->snd_recover_prev = tp->snd_recover;
892 } else {
893 /*
894 * Timestamp option is not supported on this connection.
895 * Record ssthresh and cwnd so they can
896 * be recovered if this turns out to be a "bad" retransmit.
897 * A retransmit is considered "bad" if an ACK for this
898 * segment is received within RTT/2 interval; the assumption
899 * here is that the ACK was already in flight. See
900 * "On Estimating End-to-End Network Path Properties" by
901 * Allman and Paxson for more details.
902 */
903 tp->snd_cwnd_prev = tp->snd_cwnd;
904 tp->snd_ssthresh_prev = tp->snd_ssthresh;
905 tp->snd_recover_prev = tp->snd_recover;
906 if (IN_FASTRECOVERY(tp)) {
907 tp->t_flags |= TF_WASFRECOVERY;
908 } else {
909 tp->t_flags &= ~TF_WASFRECOVERY;
910 }
911 }
912 tp->t_srtt_prev = (tp->t_srtt >> TCP_RTT_SHIFT) + 2;
913 tp->t_rttvar_prev = (tp->t_rttvar >> TCP_RTTVAR_SHIFT);
914 tp->t_flagsext &= ~(TF_RECOMPUTE_RTT);
915 }
916
917 /*
918 * Revert to the older segment size if there is an indication that PMTU
919 * blackhole detection was not needed.
920 */
921 void
tcp_pmtud_revert_segment_size(struct tcpcb * tp)922 tcp_pmtud_revert_segment_size(struct tcpcb *tp)
923 {
924 int32_t optlen;
925
926 VERIFY(tp->t_pmtud_saved_maxopd > 0);
927 tp->t_flags |= TF_PMTUD;
928 tp->t_flags &= ~TF_BLACKHOLE;
929 optlen = tp->t_maxopd - tp->t_maxseg;
930 tp->t_maxopd = tp->t_pmtud_saved_maxopd;
931 tp->t_maxseg = tp->t_maxopd - optlen;
932
933 /*
934 * Reset the slow-start flight size as it
935 * may depend on the new MSS
936 */
937 if (CC_ALGO(tp)->cwnd_init != NULL) {
938 CC_ALGO(tp)->cwnd_init(tp);
939 }
940
941 if (TCP_USE_RLEDBAT(tp, tp->t_inpcb->inp_socket) &&
942 tcp_cc_rledbat.rwnd_init != NULL) {
943 tcp_cc_rledbat.rwnd_init(tp);
944 }
945
946 tp->t_pmtud_start_ts = 0;
947 tcpstat.tcps_pmtudbh_reverted++;
948
949 /* change MSS according to recommendation, if there was one */
950 tcp_update_mss_locked(tp->t_inpcb->inp_socket, NULL);
951 }
952
953 static uint32_t
tcp_pmtud_black_holed_next_mss(struct tcpcb * tp)954 tcp_pmtud_black_holed_next_mss(struct tcpcb *tp)
955 {
956 /* Reduce the MSS to intermediary value */
957 if (tp->t_maxopd > tcp_pmtud_black_hole_mss) {
958 return tcp_pmtud_black_hole_mss;
959 } else {
960 if (tp->t_inpcb->inp_vflag & INP_IPV4) {
961 return tcp_mssdflt;
962 } else {
963 return tcp_v6mssdflt;
964 }
965 }
966 }
967
968 /*
969 * Send a packet designed to force a response
970 * if the peer is up and reachable:
971 * either an ACK if the connection is still alive,
972 * or an RST if the peer has closed the connection
973 * due to timeout or reboot.
974 * Using sequence number tp->snd_una-1
975 * causes the transmitted zero-length segment
976 * to lie outside the receive window;
977 * by the protocol spec, this requires the
978 * correspondent TCP to respond.
979 */
980 static bool
tcp_send_keep_alive(struct tcpcb * tp)981 tcp_send_keep_alive(struct tcpcb *tp)
982 {
983 struct tcptemp *t_template;
984
985 tcpstat.tcps_keepprobe++;
986 t_template = tcp_maketemplate(tp);
987 if (t_template != NULL) {
988 struct inpcb *inp = tp->t_inpcb;
989 struct tcp_respond_args tra;
990
991 bzero(&tra, sizeof(tra));
992 tra.nocell = INP_NO_CELLULAR(inp);
993 tra.noexpensive = INP_NO_EXPENSIVE(inp);
994 tra.noconstrained = INP_NO_CONSTRAINED(inp);
995 tra.awdl_unrestricted = INP_AWDL_UNRESTRICTED(inp);
996 tra.intcoproc_allowed = INP_INTCOPROC_ALLOWED(inp);
997 tra.keep_alive = 1;
998 if (tp->t_inpcb->inp_flags & INP_BOUND_IF) {
999 tra.ifscope = tp->t_inpcb->inp_boundifp->if_index;
1000 } else {
1001 tra.ifscope = IFSCOPE_NONE;
1002 }
1003 tcp_respond(tp, t_template->tt_ipgen,
1004 &t_template->tt_t, (struct mbuf *)NULL,
1005 tp->rcv_nxt, tp->snd_una - 1, 0, &tra);
1006 (void) m_free(dtom(t_template));
1007 return true;
1008 } else {
1009 return false;
1010 }
1011 }
1012
1013 /*
1014 * TCP timer processing.
1015 */
1016 struct tcpcb *
tcp_timers(struct tcpcb * tp,int timer)1017 tcp_timers(struct tcpcb *tp, int timer)
1018 {
1019 int32_t rexmt, optlen = 0, idle_time = 0;
1020 struct socket *so;
1021 #if TCPDEBUG
1022 int ostate;
1023 #endif
1024 u_int64_t accsleep_ms;
1025 u_int64_t last_sleep_ms = 0;
1026
1027 so = tp->t_inpcb->inp_socket;
1028 idle_time = tcp_now - tp->t_rcvtime;
1029
1030 switch (timer) {
1031 /*
1032 * 2 MSL timeout in shutdown went off. If we're closed but
1033 * still waiting for peer to close and connection has been idle
1034 * too long, or if 2MSL time is up from TIME_WAIT or FIN_WAIT_2,
1035 * delete connection control block.
1036 * Otherwise, (this case shouldn't happen) check again in a bit
1037 * we keep the socket in the main list in that case.
1038 */
1039 case TCPT_2MSL:
1040 tcp_free_sackholes(tp);
1041 if (tp->t_state != TCPS_TIME_WAIT &&
1042 tp->t_state != TCPS_FIN_WAIT_2 &&
1043 ((idle_time > 0) && (idle_time < TCP_CONN_MAXIDLE(tp)))) {
1044 tp->t_timer[TCPT_2MSL] = OFFSET_FROM_START(tp,
1045 (u_int32_t)TCP_CONN_KEEPINTVL(tp));
1046 } else {
1047 if (tp->t_state == TCPS_FIN_WAIT_2) {
1048 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1049 "FIN wait timeout drop");
1050 tcpstat.tcps_fin_timeout_drops++;
1051 tp = tcp_drop(tp, 0);
1052 } else {
1053 tp = tcp_close(tp);
1054 }
1055 return tp;
1056 }
1057 break;
1058
1059 /*
1060 * Retransmission timer went off. Message has not
1061 * been acked within retransmit interval. Back off
1062 * to a longer retransmit interval and retransmit one segment.
1063 */
1064 case TCPT_REXMT:
1065 absolutetime_to_nanoseconds(mach_absolutetime_asleep,
1066 &accsleep_ms);
1067 accsleep_ms = accsleep_ms / 1000000UL;
1068 if (accsleep_ms > tp->t_accsleep_ms) {
1069 last_sleep_ms = accsleep_ms - tp->t_accsleep_ms;
1070 }
1071 /*
1072 * Drop a connection in the retransmit timer
1073 * 1. If we have retransmitted more than TCP_MAXRXTSHIFT
1074 * times
1075 * 2. If the time spent in this retransmission episode is
1076 * more than the time limit set with TCP_RXT_CONNDROPTIME
1077 * socket option
1078 * 3. If TCP_RXT_FINDROP socket option was set and
1079 * we have already retransmitted the FIN 3 times without
1080 * receiving an ack
1081 */
1082 if (++tp->t_rxtshift > TCP_MAXRXTSHIFT ||
1083 (tp->t_rxt_conndroptime > 0 && tp->t_rxtstart > 0 &&
1084 (tcp_now - tp->t_rxtstart) >= tp->t_rxt_conndroptime) ||
1085 ((tp->t_flagsext & TF_RXTFINDROP) != 0 &&
1086 (tp->t_flags & TF_SENTFIN) != 0 && tp->t_rxtshift >= 4) ||
1087 (tp->t_rxtshift > 4 && last_sleep_ms >= TCP_SLEEP_TOO_LONG)) {
1088 if (tp->t_state == TCPS_ESTABLISHED &&
1089 tp->t_rxt_minimum_timeout > 0) {
1090 /*
1091 * Avoid dropping a connection if minimum
1092 * timeout is set and that time did not
1093 * pass. We will retry sending
1094 * retransmissions at the maximum interval
1095 */
1096 if (TSTMP_LT(tcp_now, (tp->t_rxtstart +
1097 tp->t_rxt_minimum_timeout))) {
1098 tp->t_rxtshift = TCP_MAXRXTSHIFT - 1;
1099 goto retransmit_packet;
1100 }
1101 }
1102 if ((tp->t_flagsext & TF_RXTFINDROP) != 0) {
1103 tcpstat.tcps_rxtfindrop++;
1104 } else if (last_sleep_ms >= TCP_SLEEP_TOO_LONG) {
1105 tcpstat.tcps_drop_after_sleep++;
1106 } else {
1107 tcpstat.tcps_timeoutdrop++;
1108 }
1109 if (tp->t_rxtshift >= TCP_MAXRXTSHIFT) {
1110 if (TCP_ECN_ENABLED(tp)) {
1111 INP_INC_IFNET_STAT(tp->t_inpcb,
1112 ecn_on.rxmit_drop);
1113 } else {
1114 INP_INC_IFNET_STAT(tp->t_inpcb,
1115 ecn_off.rxmit_drop);
1116 }
1117 }
1118 tp->t_rxtshift = TCP_MAXRXTSHIFT;
1119 soevent(so,
1120 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1121
1122 if (TCP_ECN_ENABLED(tp) &&
1123 tp->t_state == TCPS_ESTABLISHED) {
1124 tcp_heuristic_ecn_droprxmt(tp);
1125 }
1126
1127 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1128 "retransmission timeout drop");
1129 tp = tcp_drop(tp, tp->t_softerror ?
1130 tp->t_softerror : ETIMEDOUT);
1131
1132 break;
1133 }
1134 retransmit_packet:
1135 tcpstat.tcps_rexmttimeo++;
1136 tp->t_accsleep_ms = accsleep_ms;
1137
1138 if (tp->t_rxtshift == 1 &&
1139 tp->t_state == TCPS_ESTABLISHED) {
1140 /* Set the time at which retransmission started. */
1141 tp->t_rxtstart = tcp_now;
1142
1143 /*
1144 * if this is the first retransmit timeout, save
1145 * the state so that we can recover if the timeout
1146 * is spurious.
1147 */
1148 tcp_rexmt_save_state(tp);
1149 tcp_ccdbg_trace(tp, NULL, TCP_CC_FIRST_REXMT);
1150 }
1151 #if MPTCP
1152 if ((tp->t_rxtshift >= mptcp_fail_thresh) &&
1153 (tp->t_state == TCPS_ESTABLISHED) &&
1154 (tp->t_mpflags & TMPF_MPTCP_TRUE)) {
1155 mptcp_act_on_txfail(so);
1156 }
1157
1158 if (TCPS_HAVEESTABLISHED(tp->t_state) &&
1159 (so->so_flags & SOF_MP_SUBFLOW)) {
1160 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1161
1162 if (mpte->mpte_svctype == MPTCP_SVCTYPE_HANDOVER ||
1163 mpte->mpte_svctype == MPTCP_SVCTYPE_PURE_HANDOVER) {
1164 mptcp_check_subflows_and_add(mpte);
1165 }
1166 }
1167 #endif /* MPTCP */
1168
1169 if (tp->t_adaptive_wtimo > 0 &&
1170 tp->t_rxtshift > tp->t_adaptive_wtimo &&
1171 TCPS_HAVEESTABLISHED(tp->t_state)) {
1172 /* Send an event to the application */
1173 soevent(so,
1174 (SO_FILT_HINT_LOCKED |
1175 SO_FILT_HINT_ADAPTIVE_WTIMO));
1176 }
1177
1178 /*
1179 * If this is a retransmit timeout after PTO, the PTO
1180 * was not effective
1181 */
1182 if (tp->t_flagsext & TF_SENT_TLPROBE) {
1183 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1184 tcpstat.tcps_rto_after_pto++;
1185 }
1186
1187 if (tp->t_flagsext & TF_DELAY_RECOVERY) {
1188 /*
1189 * Retransmit timer fired before entering recovery
1190 * on a connection with packet re-ordering. This
1191 * suggests that the reordering metrics computed
1192 * are not accurate.
1193 */
1194 tp->t_reorderwin = 0;
1195 tp->t_timer[TCPT_DELAYFR] = 0;
1196 tp->t_flagsext &= ~(TF_DELAY_RECOVERY);
1197 }
1198
1199 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1200 tp->t_state == TCPS_SYN_RECEIVED) {
1201 tcp_disable_tfo(tp);
1202 }
1203
1204 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1205 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1206 (tp->t_tfo_stats & TFO_S_SYN_DATA_SENT) &&
1207 !(tp->t_tfo_flags & TFO_F_NO_SNDPROBING) &&
1208 ((tp->t_state != TCPS_SYN_SENT && tp->t_rxtshift > 1) ||
1209 tp->t_rxtshift > 4)) {
1210 /*
1211 * For regular retransmissions, a first one is being
1212 * done for tail-loss probe.
1213 * Thus, if rxtshift > 1, this means we have sent the segment
1214 * a total of 3 times.
1215 *
1216 * If we are in SYN-SENT state, then there is no tail-loss
1217 * probe thus we have to let rxtshift go up to 3.
1218 */
1219 tcp_heuristic_tfo_middlebox(tp);
1220
1221 so->so_error = ENODATA;
1222 soevent(so,
1223 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1224 sorwakeup(so);
1225 sowwakeup(so);
1226
1227 tp->t_tfo_stats |= TFO_S_SEND_BLACKHOLE;
1228 tcpstat.tcps_tfo_sndblackhole++;
1229 }
1230
1231 if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1232 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1233 (tp->t_tfo_stats & TFO_S_SYN_DATA_ACKED) &&
1234 tp->t_rxtshift > 3) {
1235 if (TSTMP_GT(tp->t_sndtime - 10 * TCP_RETRANSHZ, tp->t_rcvtime)) {
1236 tcp_heuristic_tfo_middlebox(tp);
1237
1238 so->so_error = ENODATA;
1239 soevent(so,
1240 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1241 sorwakeup(so);
1242 sowwakeup(so);
1243 }
1244 }
1245
1246 if (tp->t_state == TCPS_SYN_SENT) {
1247 rexmt = TCP_REXMTVAL(tp) * tcp_syn_backoff[tp->t_rxtshift];
1248 tp->t_stat.synrxtshift = tp->t_rxtshift;
1249 tp->t_stat.rxmitsyns++;
1250
1251 /* When retransmitting, disable TFO */
1252 if (tfo_enabled(tp) &&
1253 !(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE)) {
1254 tcp_disable_tfo(tp);
1255 tp->t_tfo_flags |= TFO_F_SYN_LOSS;
1256 }
1257 } else {
1258 rexmt = TCP_REXMTVAL(tp) * tcp_backoff[tp->t_rxtshift];
1259 }
1260
1261 TCPT_RANGESET(tp->t_rxtcur, rexmt, tp->t_rttmin, TCPTV_REXMTMAX,
1262 TCP_ADD_REXMTSLOP(tp));
1263 tp->t_timer[TCPT_REXMT] = OFFSET_FROM_START(tp, tp->t_rxtcur);
1264
1265 TCP_LOG_RTT_INFO(tp);
1266
1267 if (INP_WAIT_FOR_IF_FEEDBACK(tp->t_inpcb)) {
1268 goto fc_output;
1269 }
1270
1271 tcp_free_sackholes(tp);
1272 /*
1273 * Check for potential Path MTU Discovery Black Hole
1274 */
1275 if (tcp_pmtud_black_hole_detect &&
1276 !(tp->t_flagsext & TF_NOBLACKHOLE_DETECTION) &&
1277 (tp->t_state == TCPS_ESTABLISHED)) {
1278 if ((tp->t_flags & TF_PMTUD) &&
1279 tp->t_pmtud_lastseg_size > tcp_pmtud_black_holed_next_mss(tp) &&
1280 tp->t_rxtshift == 2) {
1281 /*
1282 * Enter Path MTU Black-hole Detection mechanism:
1283 * - Disable Path MTU Discovery (IP "DF" bit).
1284 * - Reduce MTU to lower value than what we
1285 * negotiated with the peer.
1286 */
1287 /* Disable Path MTU Discovery for now */
1288 tp->t_flags &= ~TF_PMTUD;
1289 /* Record that we may have found a black hole */
1290 tp->t_flags |= TF_BLACKHOLE;
1291 optlen = tp->t_maxopd - tp->t_maxseg;
1292 /* Keep track of previous MSS */
1293 tp->t_pmtud_saved_maxopd = tp->t_maxopd;
1294 tp->t_pmtud_start_ts = tcp_now;
1295 if (tp->t_pmtud_start_ts == 0) {
1296 tp->t_pmtud_start_ts++;
1297 }
1298 /* Reduce the MSS to intermediary value */
1299 tp->t_maxopd = tcp_pmtud_black_holed_next_mss(tp);
1300 tp->t_maxseg = tp->t_maxopd - optlen;
1301
1302 /*
1303 * Reset the slow-start flight size
1304 * as it may depend on the new MSS
1305 */
1306 if (CC_ALGO(tp)->cwnd_init != NULL) {
1307 CC_ALGO(tp)->cwnd_init(tp);
1308 }
1309 tp->snd_cwnd = tp->t_maxseg;
1310
1311 if (TCP_USE_RLEDBAT(tp, so) &&
1312 tcp_cc_rledbat.rwnd_init != NULL) {
1313 tcp_cc_rledbat.rwnd_init(tp);
1314 }
1315 }
1316 /*
1317 * If further retransmissions are still
1318 * unsuccessful with a lowered MTU, maybe this
1319 * isn't a Black Hole and we restore the previous
1320 * MSS and blackhole detection flags.
1321 */
1322 else {
1323 if ((tp->t_flags & TF_BLACKHOLE) &&
1324 (tp->t_rxtshift > 4)) {
1325 tcp_pmtud_revert_segment_size(tp);
1326 tp->snd_cwnd = tp->t_maxseg;
1327 }
1328 }
1329 }
1330
1331 /*
1332 * Disable rfc1323 and rfc1644 if we haven't got any
1333 * response to our SYN (after we reach the threshold)
1334 * to work-around some broken terminal servers (most of
1335 * which have hopefully been retired) that have bad VJ
1336 * header compression code which trashes TCP segments
1337 * containing unknown-to-them TCP options.
1338 * Do this only on non-local connections.
1339 */
1340 if (tp->t_state == TCPS_SYN_SENT &&
1341 tp->t_rxtshift == tcp_broken_peer_syn_rxmit_thres) {
1342 tp->t_flags &= ~(TF_REQ_SCALE | TF_REQ_TSTMP | TF_REQ_CC);
1343 }
1344
1345 /*
1346 * If losing, let the lower level know and try for
1347 * a better route. Also, if we backed off this far,
1348 * our srtt estimate is probably bogus. Clobber it
1349 * so we'll take the next rtt measurement as our srtt;
1350 * move the current srtt into rttvar to keep the current
1351 * retransmit times until then.
1352 */
1353 if (tp->t_rxtshift > TCP_MAXRXTSHIFT / 4) {
1354 if (!(tp->t_inpcb->inp_vflag & INP_IPV4)) {
1355 in6_losing(tp->t_inpcb);
1356 } else {
1357 in_losing(tp->t_inpcb);
1358 }
1359 tp->t_rttvar += (tp->t_srtt >> TCP_RTT_SHIFT);
1360 tp->t_srtt = 0;
1361 }
1362 tp->snd_nxt = tp->snd_una;
1363 /*
1364 * Note: We overload snd_recover to function also as the
1365 * snd_last variable described in RFC 2582
1366 */
1367 tp->snd_recover = tp->snd_max;
1368 /*
1369 * Force a segment to be sent.
1370 */
1371 tp->t_flags |= TF_ACKNOW;
1372
1373 /* If timing a segment in this window, stop the timer */
1374 tp->t_rtttime = 0;
1375
1376 if (!IN_FASTRECOVERY(tp) && tp->t_rxtshift == 1) {
1377 tcpstat.tcps_tailloss_rto++;
1378 }
1379
1380
1381 /*
1382 * RFC 5681 says: when a TCP sender detects segment loss
1383 * using retransmit timer and the given segment has already
1384 * been retransmitted by way of the retransmission timer at
1385 * least once, the value of ssthresh is held constant
1386 */
1387 if (tp->t_rxtshift == 1 &&
1388 CC_ALGO(tp)->after_timeout != NULL) {
1389 CC_ALGO(tp)->after_timeout(tp);
1390 /*
1391 * CWR notifications are to be sent on new data
1392 * right after Fast Retransmits and ECE
1393 * notification receipts.
1394 */
1395 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1396 tp->ecn_flags |= TE_SENDCWR;
1397 }
1398 }
1399
1400 EXIT_FASTRECOVERY(tp);
1401
1402 /* Exit cwnd non validated phase */
1403 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
1404
1405
1406 fc_output:
1407 tcp_ccdbg_trace(tp, NULL, TCP_CC_REXMT_TIMEOUT);
1408
1409 (void) tcp_output(tp);
1410 break;
1411
1412 /*
1413 * Persistance timer into zero window.
1414 * Force a byte to be output, if possible.
1415 */
1416 case TCPT_PERSIST:
1417 tcpstat.tcps_persisttimeo++;
1418 /*
1419 * Hack: if the peer is dead/unreachable, we do not
1420 * time out if the window is closed. After a full
1421 * backoff, drop the connection if the idle time
1422 * (no responses to probes) reaches the maximum
1423 * backoff that we would use if retransmitting.
1424 *
1425 * Drop the connection if we reached the maximum allowed time for
1426 * Zero Window Probes without a non-zero update from the peer.
1427 * See rdar://5805356
1428 */
1429 if ((tp->t_rxtshift == TCP_MAXRXTSHIFT &&
1430 (idle_time >= tcp_maxpersistidle ||
1431 idle_time >= TCP_REXMTVAL(tp) * tcp_totbackoff)) ||
1432 ((tp->t_persist_stop != 0) &&
1433 TSTMP_LEQ(tp->t_persist_stop, tcp_now))) {
1434 TCP_LOG_DROP_PCB(NULL, NULL, tp, false, "persist timeout drop");
1435 tcpstat.tcps_persistdrop++;
1436 soevent(so,
1437 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1438 tp = tcp_drop(tp, ETIMEDOUT);
1439 break;
1440 }
1441 tcp_setpersist(tp);
1442 tp->t_flagsext |= TF_FORCE;
1443 (void) tcp_output(tp);
1444 tp->t_flagsext &= ~TF_FORCE;
1445 break;
1446
1447 /*
1448 * Keep-alive timer went off; send something
1449 * or drop connection if idle for too long.
1450 */
1451 case TCPT_KEEP:
1452 #if FLOW_DIVERT
1453 if (tp->t_inpcb->inp_socket->so_flags & SOF_FLOW_DIVERT) {
1454 break;
1455 }
1456 #endif /* FLOW_DIVERT */
1457
1458 tcpstat.tcps_keeptimeo++;
1459 #if MPTCP
1460 /*
1461 * Regular TCP connections do not send keepalives after closing
1462 * MPTCP must not also, after sending Data FINs.
1463 */
1464 struct mptcb *mp_tp = tptomptp(tp);
1465 if ((tp->t_mpflags & TMPF_MPTCP_TRUE) &&
1466 (tp->t_state > TCPS_ESTABLISHED)) {
1467 goto dropit;
1468 } else if (mp_tp != NULL) {
1469 if ((mptcp_ok_to_keepalive(mp_tp) == 0)) {
1470 goto dropit;
1471 }
1472 }
1473 #endif /* MPTCP */
1474 if (tp->t_state < TCPS_ESTABLISHED) {
1475 goto dropit;
1476 }
1477 if ((always_keepalive ||
1478 (tp->t_inpcb->inp_socket->so_options & SO_KEEPALIVE) ||
1479 (tp->t_flagsext & TF_DETECT_READSTALL) ||
1480 (tp->t_tfo_probe_state == TFO_PROBE_PROBING)) &&
1481 (tp->t_state <= TCPS_CLOSING || tp->t_state == TCPS_FIN_WAIT_2)) {
1482 if (idle_time >= TCP_CONN_KEEPIDLE(tp) + TCP_CONN_MAXIDLE(tp)) {
1483 TCP_LOG_DROP_PCB(NULL, NULL, tp, false,
1484 "keep alive timeout drop");
1485 goto dropit;
1486 }
1487
1488 if (tcp_send_keep_alive(tp)) {
1489 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1490 tp->t_rtimo_probes++;
1491 }
1492
1493 TCP_LOG_KEEP_ALIVE(tp, idle_time);
1494 }
1495
1496 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1497 TCP_CONN_KEEPINTVL(tp));
1498 } else {
1499 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
1500 TCP_CONN_KEEPIDLE(tp));
1501 }
1502 if (tp->t_flagsext & TF_DETECT_READSTALL) {
1503 struct ifnet *outifp = tp->t_inpcb->inp_last_outifp;
1504 bool reenable_probe = false;
1505 /*
1506 * The keep alive packets sent to detect a read
1507 * stall did not get a response from the
1508 * peer. Generate more keep-alives to confirm this.
1509 * If the number of probes sent reaches the limit,
1510 * generate an event.
1511 */
1512 if (tp->t_adaptive_rtimo > 0) {
1513 if (tp->t_rtimo_probes > tp->t_adaptive_rtimo) {
1514 /* Generate an event */
1515 soevent(so,
1516 (SO_FILT_HINT_LOCKED |
1517 SO_FILT_HINT_ADAPTIVE_RTIMO));
1518 tcp_keepalive_reset(tp);
1519 } else {
1520 reenable_probe = true;
1521 }
1522 } else if (outifp != NULL &&
1523 (outifp->if_eflags & IFEF_PROBE_CONNECTIVITY) &&
1524 tp->t_rtimo_probes <= TCP_CONNECTIVITY_PROBES_MAX) {
1525 reenable_probe = true;
1526 } else {
1527 tp->t_flagsext &= ~TF_DETECT_READSTALL;
1528 }
1529 if (reenable_probe) {
1530 int ind = min(tp->t_rtimo_probes,
1531 TCP_MAXRXTSHIFT);
1532 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(
1533 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp));
1534 }
1535 }
1536 if (tp->t_tfo_probe_state == TFO_PROBE_PROBING) {
1537 int ind;
1538
1539 tp->t_tfo_probes++;
1540 ind = min(tp->t_tfo_probes, TCP_MAXRXTSHIFT);
1541
1542 /*
1543 * We take the minimum among the time set by true
1544 * keepalive (see above) and the backoff'd RTO. That
1545 * way we backoff in case of packet-loss but will never
1546 * timeout slower than regular keepalive due to the
1547 * backing off.
1548 */
1549 tp->t_timer[TCPT_KEEP] = min(OFFSET_FROM_START(
1550 tp, tcp_backoff[ind] * TCP_REXMTVAL(tp)),
1551 tp->t_timer[TCPT_KEEP]);
1552 } else if (!(tp->t_flagsext & TF_FASTOPEN_FORCE_ENABLE) &&
1553 !(tp->t_tfo_flags & TFO_F_HEURISTIC_DONE) &&
1554 tp->t_tfo_probe_state == TFO_PROBE_WAIT_DATA) {
1555 /* Still no data! Let's assume a TFO-error and err out... */
1556 tcp_heuristic_tfo_middlebox(tp);
1557
1558 so->so_error = ENODATA;
1559 soevent(so,
1560 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_MP_SUB_ERROR));
1561 sorwakeup(so);
1562 tp->t_tfo_stats |= TFO_S_RECV_BLACKHOLE;
1563 tcpstat.tcps_tfo_blackhole++;
1564 }
1565 break;
1566 case TCPT_DELACK:
1567 if (tcp_delack_enabled && (tp->t_flags & TF_DELACK)) {
1568 tp->t_flags &= ~TF_DELACK;
1569 tp->t_timer[TCPT_DELACK] = 0;
1570 tp->t_flags |= TF_ACKNOW;
1571
1572 /*
1573 * If delayed ack timer fired while stretching
1574 * acks, count the number of times the streaming
1575 * detection was not correct. If this exceeds a
1576 * threshold, disable strech ack on this
1577 * connection
1578 *
1579 * Also, go back to acking every other packet.
1580 */
1581 if ((tp->t_flags & TF_STRETCHACK)) {
1582 if (tp->t_unacksegs > 1 &&
1583 tp->t_unacksegs < maxseg_unacked) {
1584 tp->t_stretchack_delayed++;
1585 }
1586
1587 if (tp->t_stretchack_delayed >
1588 TCP_STRETCHACK_DELAY_THRESHOLD) {
1589 tp->t_flagsext |= TF_DISABLE_STRETCHACK;
1590 /*
1591 * Note the time at which stretch
1592 * ack was disabled automatically
1593 */
1594 tp->rcv_nostrack_ts = tcp_now;
1595 tcpstat.tcps_nostretchack++;
1596 tp->t_stretchack_delayed = 0;
1597 tp->rcv_nostrack_pkts = 0;
1598 }
1599 tcp_reset_stretch_ack(tp);
1600 }
1601 tp->t_forced_acks = TCP_FORCED_ACKS_COUNT;
1602
1603 /*
1604 * If we are measuring inter packet arrival jitter
1605 * for throttling a connection, this delayed ack
1606 * might be the reason for accumulating some
1607 * jitter. So let's restart the measurement.
1608 */
1609 CLEAR_IAJ_STATE(tp);
1610
1611 tcpstat.tcps_delack++;
1612 tp->t_stat.delayed_acks_sent++;
1613 (void) tcp_output(tp);
1614 }
1615 break;
1616
1617 #if MPTCP
1618 case TCPT_JACK_RXMT:
1619 if ((tp->t_state == TCPS_ESTABLISHED) &&
1620 (tp->t_mpflags & TMPF_PREESTABLISHED) &&
1621 (tp->t_mpflags & TMPF_JOINED_FLOW)) {
1622 if (++tp->t_mprxtshift > TCP_MAXRXTSHIFT) {
1623 tcpstat.tcps_timeoutdrop++;
1624 soevent(so,
1625 (SO_FILT_HINT_LOCKED |
1626 SO_FILT_HINT_TIMEOUT));
1627 tp = tcp_drop(tp, tp->t_softerror ?
1628 tp->t_softerror : ETIMEDOUT);
1629 break;
1630 }
1631 tcpstat.tcps_join_rxmts++;
1632 tp->t_mpflags |= TMPF_SND_JACK;
1633 tp->t_flags |= TF_ACKNOW;
1634
1635 /*
1636 * No backoff is implemented for simplicity for this
1637 * corner case.
1638 */
1639 (void) tcp_output(tp);
1640 }
1641 break;
1642 case TCPT_CELLICON:
1643 {
1644 struct mptses *mpte = tptomptp(tp)->mpt_mpte;
1645
1646 tp->t_timer[TCPT_CELLICON] = 0;
1647
1648 if (mpte->mpte_cellicon_increments == 0) {
1649 /* Cell-icon not set by this connection */
1650 break;
1651 }
1652
1653 if (TSTMP_LT(mpte->mpte_last_cellicon_set + MPTCP_CELLICON_TOGGLE_RATE, tcp_now)) {
1654 mptcp_unset_cellicon(mpte, NULL, 1);
1655 }
1656
1657 if (mpte->mpte_cellicon_increments) {
1658 tp->t_timer[TCPT_CELLICON] = OFFSET_FROM_START(tp, MPTCP_CELLICON_TOGGLE_RATE);
1659 }
1660
1661 break;
1662 }
1663 #endif /* MPTCP */
1664
1665 case TCPT_PTO:
1666 {
1667 int32_t ret = 0;
1668
1669 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1670 tp->t_flagsext &= ~(TF_SENT_TLPROBE);
1671 }
1672 /*
1673 * Check if the connection is in the right state to
1674 * send a probe
1675 */
1676 if ((tp->t_state != TCPS_ESTABLISHED ||
1677 tp->t_rxtshift > 0 ||
1678 tp->snd_max == tp->snd_una ||
1679 !SACK_ENABLED(tp) ||
1680 (tcp_do_better_lr != 1 && !TAILQ_EMPTY(&tp->snd_holes)) ||
1681 IN_FASTRECOVERY(tp)) &&
1682 !(tp->t_flagsext & TF_IF_PROBING)) {
1683 break;
1684 }
1685
1686 /*
1687 * When the interface state is changed explicitly reset the retransmission
1688 * timer state for both SYN and data packets because we do not want to
1689 * wait unnecessarily or timeout too quickly if the link characteristics
1690 * have changed drastically
1691 */
1692 if (tp->t_flagsext & TF_IF_PROBING) {
1693 tp->t_rxtshift = 0;
1694 if (tp->t_state == TCPS_SYN_SENT) {
1695 tp->t_stat.synrxtshift = tp->t_rxtshift;
1696 }
1697 /*
1698 * Reset to the the default RTO
1699 */
1700 tp->t_srtt = TCPTV_SRTTBASE;
1701 tp->t_rttvar =
1702 ((TCPTV_RTOBASE - TCPTV_SRTTBASE) << TCP_RTTVAR_SHIFT) / 4;
1703 tp->t_rttmin = tp->t_flags & TF_LOCAL ? tcp_TCPTV_MIN :
1704 TCPTV_REXMTMIN;
1705 TCPT_RANGESET(tp->t_rxtcur, TCP_REXMTVAL(tp),
1706 tp->t_rttmin, TCPTV_REXMTMAX, TCP_ADD_REXMTSLOP(tp));
1707 TCP_LOG_RTT_INFO(tp);
1708 }
1709
1710 if (tp->t_state == TCPS_SYN_SENT) {
1711 /*
1712 * The PTO for SYN_SENT reinitializes TCP as if it was a fresh
1713 * connection attempt
1714 */
1715 tp->snd_nxt = tp->snd_una;
1716 /*
1717 * Note: We overload snd_recover to function also as the
1718 * snd_last variable described in RFC 2582
1719 */
1720 tp->snd_recover = tp->snd_max;
1721 /*
1722 * Force a segment to be sent.
1723 */
1724 tp->t_flags |= TF_ACKNOW;
1725
1726 /* If timing a segment in this window, stop the timer */
1727 tp->t_rtttime = 0;
1728 } else {
1729 int32_t snd_len;
1730
1731 /*
1732 * If there is no new data to send or if the
1733 * connection is limited by receive window then
1734 * retransmit the last segment, otherwise send
1735 * new data.
1736 */
1737 snd_len = min(so->so_snd.sb_cc, tp->snd_wnd)
1738 - (tp->snd_max - tp->snd_una);
1739 if (snd_len > 0) {
1740 tp->snd_nxt = tp->snd_max;
1741 } else {
1742 snd_len = min((tp->snd_max - tp->snd_una),
1743 tp->t_maxseg);
1744 tp->snd_nxt = tp->snd_max - snd_len;
1745 }
1746 }
1747
1748 tcpstat.tcps_pto++;
1749 if (tp->t_flagsext & TF_IF_PROBING) {
1750 tcpstat.tcps_probe_if++;
1751 }
1752
1753 /* If timing a segment in this window, stop the timer */
1754 tp->t_rtttime = 0;
1755 /* Note that tail loss probe is being sent. Exclude IF probe */
1756 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1757 tp->t_flagsext |= TF_SENT_TLPROBE;
1758 tp->t_tlpstart = tcp_now;
1759 }
1760
1761 tp->snd_cwnd += tp->t_maxseg;
1762 /*
1763 * When tail-loss-probe fires, we reset the RTO timer, because
1764 * a probe just got sent, so we are good to push out the timer.
1765 *
1766 * Set to 0 to ensure that tcp_output() will reschedule it
1767 */
1768 tp->t_timer[TCPT_REXMT] = 0;
1769 ret = tcp_output(tp);
1770
1771 #if (DEBUG || DEVELOPMENT)
1772 if ((tp->t_flagsext & TF_IF_PROBING) &&
1773 ((IFNET_IS_COMPANION_LINK(tp->t_inpcb->inp_last_outifp)) ||
1774 tp->t_state == TCPS_SYN_SENT)) {
1775 if (ret == 0 && tcp_probe_if_fix_port > 0 &&
1776 tcp_probe_if_fix_port <= IPPORT_HILASTAUTO) {
1777 tp->t_timer[TCPT_REXMT] = 0;
1778 tcp_set_lotimer_index(tp);
1779 }
1780
1781 os_log(OS_LOG_DEFAULT,
1782 "%s: sent %s probe for %u > %u on interface %s"
1783 " (%u) %s(%d)",
1784 __func__,
1785 tp->t_state == TCPS_SYN_SENT ? "SYN" : "data",
1786 ntohs(tp->t_inpcb->inp_lport),
1787 ntohs(tp->t_inpcb->inp_fport),
1788 if_name(tp->t_inpcb->inp_last_outifp),
1789 tp->t_inpcb->inp_last_outifp->if_index,
1790 ret == 0 ? "succeeded" :"failed", ret);
1791 }
1792 #endif /* DEBUG || DEVELOPMENT */
1793
1794 /*
1795 * When there is data (or a SYN) to send, the above call to
1796 * tcp_output() should have armed either the REXMT or the
1797 * PERSIST timer. If it didn't, something is wrong and this
1798 * connection would idle around forever. Let's make sure that
1799 * at least the REXMT timer is set.
1800 */
1801 if (tp->t_timer[TCPT_REXMT] == 0 && tp->t_timer[TCPT_PERSIST] == 0 &&
1802 (tp->t_inpcb->inp_socket->so_snd.sb_cc != 0 || tp->t_state == TCPS_SYN_SENT ||
1803 tp->t_state == TCPS_SYN_RECEIVED)) {
1804 tp->t_timer[TCPT_REXMT] =
1805 OFFSET_FROM_START(tp, tp->t_rxtcur);
1806
1807 os_log(OS_LOG_DEFAULT,
1808 "%s: tcp_output() returned %u with retransmission timer disabled "
1809 "for %u > %u in state %d, reset timer to %d",
1810 __func__, ret,
1811 ntohs(tp->t_inpcb->inp_lport),
1812 ntohs(tp->t_inpcb->inp_fport),
1813 tp->t_state,
1814 tp->t_timer[TCPT_REXMT]);
1815
1816 tcp_check_timer_state(tp);
1817 }
1818 tp->snd_cwnd -= tp->t_maxseg;
1819
1820 if (!(tp->t_flagsext & TF_IF_PROBING)) {
1821 tp->t_tlphighrxt = tp->snd_nxt;
1822 }
1823 break;
1824 }
1825 case TCPT_DELAYFR:
1826 tp->t_flagsext &= ~TF_DELAY_RECOVERY;
1827
1828 /*
1829 * Don't do anything if one of the following is true:
1830 * - the connection is already in recovery
1831 * - sequence until snd_recover has been acknowledged.
1832 * - retransmit timeout has fired
1833 */
1834 if (IN_FASTRECOVERY(tp) ||
1835 SEQ_GEQ(tp->snd_una, tp->snd_recover) ||
1836 tp->t_rxtshift > 0) {
1837 break;
1838 }
1839
1840 VERIFY(SACK_ENABLED(tp));
1841 tcp_rexmt_save_state(tp);
1842 if (CC_ALGO(tp)->pre_fr != NULL) {
1843 CC_ALGO(tp)->pre_fr(tp);
1844 if (!TCP_ACC_ECN_ON(tp) && TCP_ECN_ENABLED(tp)) {
1845 tp->ecn_flags |= TE_SENDCWR;
1846 }
1847 }
1848 ENTER_FASTRECOVERY(tp);
1849
1850 tp->t_timer[TCPT_REXMT] = 0;
1851 tcpstat.tcps_sack_recovery_episode++;
1852 tp->t_sack_recovery_episode++;
1853 tp->sack_newdata = tp->snd_nxt;
1854 tp->snd_cwnd = tp->t_maxseg;
1855 tcp_ccdbg_trace(tp, NULL, TCP_CC_ENTER_FASTRECOVERY);
1856 (void) tcp_output(tp);
1857 break;
1858
1859 dropit:
1860 tcpstat.tcps_keepdrops++;
1861 soevent(so,
1862 (SO_FILT_HINT_LOCKED | SO_FILT_HINT_TIMEOUT));
1863 tp = tcp_drop(tp, ETIMEDOUT);
1864 break;
1865 }
1866 #if TCPDEBUG
1867 if (tp->t_inpcb->inp_socket->so_options & SO_DEBUG) {
1868 tcp_trace(TA_USER, ostate, tp, (void *)0, (struct tcphdr *)0,
1869 PRU_SLOWTIMO);
1870 }
1871 #endif
1872 return tp;
1873 }
1874
1875 /* Remove a timer entry from timer list */
1876 void
tcp_remove_timer(struct tcpcb * tp)1877 tcp_remove_timer(struct tcpcb *tp)
1878 {
1879 struct tcptimerlist *listp = &tcp_timer_list;
1880
1881 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
1882 if (!(TIMER_IS_ON_LIST(tp))) {
1883 return;
1884 }
1885 lck_mtx_lock(&listp->mtx);
1886
1887 /* Check if pcb is on timer list again after acquiring the lock */
1888 if (!(TIMER_IS_ON_LIST(tp))) {
1889 lck_mtx_unlock(&listp->mtx);
1890 return;
1891 }
1892
1893 if (listp->next_te != NULL && listp->next_te == &tp->tentry) {
1894 listp->next_te = LIST_NEXT(&tp->tentry, le);
1895 }
1896
1897 LIST_REMOVE(&tp->tentry, le);
1898 tp->t_flags &= ~(TF_TIMER_ONLIST);
1899
1900 listp->entries--;
1901
1902 tp->tentry.le.le_next = NULL;
1903 tp->tentry.le.le_prev = NULL;
1904 lck_mtx_unlock(&listp->mtx);
1905 }
1906
1907 /*
1908 * Function to check if the timerlist needs to be rescheduled to run
1909 * the timer entry correctly. Basically, this is to check if we can avoid
1910 * taking the list lock.
1911 */
1912
1913 static boolean_t
need_to_resched_timerlist(u_int32_t runtime,u_int16_t mode)1914 need_to_resched_timerlist(u_int32_t runtime, u_int16_t mode)
1915 {
1916 struct tcptimerlist *listp = &tcp_timer_list;
1917 int32_t diff;
1918
1919 /*
1920 * If the list is being processed then the state of the list is
1921 * in flux. In this case always acquire the lock and set the state
1922 * correctly.
1923 */
1924 if (listp->running) {
1925 return TRUE;
1926 }
1927
1928 if (!listp->scheduled) {
1929 return TRUE;
1930 }
1931
1932 diff = timer_diff(listp->runtime, 0, runtime, 0);
1933 if (diff <= 0) {
1934 /* The list is going to run before this timer */
1935 return FALSE;
1936 } else {
1937 if (mode & TCP_TIMERLIST_10MS_MODE) {
1938 if (diff <= TCP_TIMER_10MS_QUANTUM) {
1939 return FALSE;
1940 }
1941 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
1942 if (diff <= TCP_TIMER_100MS_QUANTUM) {
1943 return FALSE;
1944 }
1945 } else {
1946 if (diff <= TCP_TIMER_500MS_QUANTUM) {
1947 return FALSE;
1948 }
1949 }
1950 }
1951 return TRUE;
1952 }
1953
1954 void
tcp_sched_timerlist(uint32_t offset)1955 tcp_sched_timerlist(uint32_t offset)
1956 {
1957 uint64_t deadline = 0;
1958 struct tcptimerlist *listp = &tcp_timer_list;
1959
1960 LCK_MTX_ASSERT(&listp->mtx, LCK_MTX_ASSERT_OWNED);
1961
1962 offset = min(offset, TCP_TIMERLIST_MAX_OFFSET);
1963 listp->runtime = tcp_now + offset;
1964 listp->schedtime = tcp_now;
1965 if (listp->runtime == 0) {
1966 listp->runtime++;
1967 offset++;
1968 }
1969
1970 clock_interval_to_deadline(offset, USEC_PER_SEC, &deadline);
1971
1972 thread_call_enter_delayed(listp->call, deadline);
1973 listp->scheduled = TRUE;
1974 }
1975
1976 /*
1977 * Function to run the timers for a connection.
1978 *
1979 * Returns the offset of next timer to be run for this connection which
1980 * can be used to reschedule the timerlist.
1981 *
1982 * te_mode is an out parameter that indicates the modes of active
1983 * timers for this connection.
1984 */
1985 u_int32_t
tcp_run_conn_timer(struct tcpcb * tp,u_int16_t * te_mode,u_int16_t probe_if_index)1986 tcp_run_conn_timer(struct tcpcb *tp, u_int16_t *te_mode,
1987 u_int16_t probe_if_index)
1988 {
1989 struct socket *so;
1990 u_int16_t i = 0, index = TCPT_NONE, lo_index = TCPT_NONE;
1991 u_int32_t timer_val, offset = 0, lo_timer = 0;
1992 int32_t diff;
1993 boolean_t needtorun[TCPT_NTIMERS];
1994 int count = 0;
1995
1996 VERIFY(tp != NULL);
1997 bzero(needtorun, sizeof(needtorun));
1998 *te_mode = 0;
1999
2000 socket_lock(tp->t_inpcb->inp_socket, 1);
2001
2002 so = tp->t_inpcb->inp_socket;
2003 /* Release the want count on inp */
2004 if (in_pcb_checkstate(tp->t_inpcb, WNT_RELEASE, 1)
2005 == WNT_STOPUSING) {
2006 if (TIMER_IS_ON_LIST(tp)) {
2007 tcp_remove_timer(tp);
2008 }
2009
2010 /* Looks like the TCP connection got closed while we
2011 * were waiting for the lock.. Done
2012 */
2013 goto done;
2014 }
2015
2016 /*
2017 * If this connection is over an interface that needs to
2018 * be probed, send probe packets to reinitiate communication.
2019 */
2020 if (TCP_IF_STATE_CHANGED(tp, probe_if_index)) {
2021 tp->t_flagsext |= TF_IF_PROBING;
2022 tcp_timers(tp, TCPT_PTO);
2023 tp->t_timer[TCPT_PTO] = 0;
2024 tp->t_flagsext &= ~TF_IF_PROBING;
2025 }
2026
2027 /*
2028 * Since the timer thread needs to wait for tcp lock, it may race
2029 * with another thread that can cancel or reschedule the timer
2030 * that is about to run. Check if we need to run anything.
2031 */
2032 if ((index = tp->tentry.index) == TCPT_NONE) {
2033 goto done;
2034 }
2035
2036 timer_val = tp->t_timer[index];
2037
2038 diff = timer_diff(tp->tentry.runtime, 0, tcp_now, 0);
2039 if (diff > 0) {
2040 if (tp->tentry.index != TCPT_NONE) {
2041 offset = diff;
2042 *(te_mode) = tp->tentry.mode;
2043 }
2044 goto done;
2045 }
2046
2047 tp->t_timer[index] = 0;
2048 if (timer_val > 0) {
2049 tp = tcp_timers(tp, index);
2050 if (tp == NULL) {
2051 goto done;
2052 }
2053 }
2054
2055 /*
2056 * Check if there are any other timers that need to be run.
2057 * While doing it, adjust the timer values wrt tcp_now.
2058 */
2059 tp->tentry.mode = 0;
2060 for (i = 0; i < TCPT_NTIMERS; ++i) {
2061 if (tp->t_timer[i] != 0) {
2062 diff = timer_diff(tp->tentry.timer_start,
2063 tp->t_timer[i], tcp_now, 0);
2064 if (diff <= 0) {
2065 needtorun[i] = TRUE;
2066 count++;
2067 } else {
2068 tp->t_timer[i] = diff;
2069 needtorun[i] = FALSE;
2070 if (lo_timer == 0 || diff < lo_timer) {
2071 lo_timer = diff;
2072 lo_index = i;
2073 }
2074 TCP_SET_TIMER_MODE(tp->tentry.mode, i);
2075 }
2076 }
2077 }
2078
2079 tp->tentry.timer_start = tcp_now;
2080 tp->tentry.index = lo_index;
2081 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2082
2083 if (tp->tentry.index != TCPT_NONE) {
2084 tp->tentry.runtime = tp->tentry.timer_start +
2085 tp->t_timer[tp->tentry.index];
2086 if (tp->tentry.runtime == 0) {
2087 tp->tentry.runtime++;
2088 }
2089 }
2090
2091 if (count > 0) {
2092 /* run any other timers outstanding at this time. */
2093 for (i = 0; i < TCPT_NTIMERS; ++i) {
2094 if (needtorun[i]) {
2095 tp->t_timer[i] = 0;
2096 tp = tcp_timers(tp, i);
2097 if (tp == NULL) {
2098 offset = 0;
2099 *(te_mode) = 0;
2100 goto done;
2101 }
2102 }
2103 }
2104 tcp_set_lotimer_index(tp);
2105 }
2106
2107 if (tp->tentry.index < TCPT_NONE) {
2108 offset = tp->t_timer[tp->tentry.index];
2109 *(te_mode) = tp->tentry.mode;
2110 }
2111
2112 done:
2113 if (tp != NULL && tp->tentry.index == TCPT_NONE) {
2114 tcp_remove_timer(tp);
2115 offset = 0;
2116 }
2117
2118 socket_unlock(so, 1);
2119 return offset;
2120 }
2121
2122 void
tcp_run_timerlist(void * arg1,void * arg2)2123 tcp_run_timerlist(void * arg1, void * arg2)
2124 {
2125 #pragma unused(arg1, arg2)
2126 struct tcptimerentry *te, *next_te;
2127 struct tcptimerlist *listp = &tcp_timer_list;
2128 struct tcpcb *tp;
2129 uint32_t next_timer = 0; /* offset of the next timer on the list */
2130 u_int16_t te_mode = 0; /* modes of all active timers in a tcpcb */
2131 u_int16_t list_mode = 0; /* cumulative of modes of all tcpcbs */
2132 uint32_t active_count = 0;
2133
2134 calculate_tcp_clock();
2135
2136 lck_mtx_lock(&listp->mtx);
2137
2138 int32_t drift = tcp_now - listp->runtime;
2139 if (drift <= 1) {
2140 tcpstat.tcps_timer_drift_le_1_ms++;
2141 } else if (drift <= 10) {
2142 tcpstat.tcps_timer_drift_le_10_ms++;
2143 } else if (drift <= 20) {
2144 tcpstat.tcps_timer_drift_le_20_ms++;
2145 } else if (drift <= 50) {
2146 tcpstat.tcps_timer_drift_le_50_ms++;
2147 } else if (drift <= 100) {
2148 tcpstat.tcps_timer_drift_le_100_ms++;
2149 } else if (drift <= 200) {
2150 tcpstat.tcps_timer_drift_le_200_ms++;
2151 } else if (drift <= 500) {
2152 tcpstat.tcps_timer_drift_le_500_ms++;
2153 } else if (drift <= 1000) {
2154 tcpstat.tcps_timer_drift_le_1000_ms++;
2155 } else {
2156 tcpstat.tcps_timer_drift_gt_1000_ms++;
2157 }
2158
2159 listp->running = TRUE;
2160
2161 LIST_FOREACH_SAFE(te, &listp->lhead, le, next_te) {
2162 uint32_t offset = 0;
2163 uint32_t runtime = te->runtime;
2164
2165 tp = TIMERENTRY_TO_TP(te);
2166
2167 /*
2168 * An interface probe may need to happen before the previously scheduled runtime
2169 */
2170 if (te->index < TCPT_NONE && TSTMP_GT(runtime, tcp_now) &&
2171 !TCP_IF_STATE_CHANGED(tp, listp->probe_if_index)) {
2172 offset = timer_diff(runtime, 0, tcp_now, 0);
2173 if (next_timer == 0 || offset < next_timer) {
2174 next_timer = offset;
2175 }
2176 list_mode |= te->mode;
2177 continue;
2178 }
2179
2180 /*
2181 * Acquire an inp wantcnt on the inpcb so that the socket
2182 * won't get detached even if tcp_close is called
2183 */
2184 if (in_pcb_checkstate(tp->t_inpcb, WNT_ACQUIRE, 0)
2185 == WNT_STOPUSING) {
2186 /*
2187 * Some how this pcb went into dead state while
2188 * on the timer list, just take it off the list.
2189 * Since the timer list entry pointers are
2190 * protected by the timer list lock, we can
2191 * do it here without the socket lock.
2192 */
2193 if (TIMER_IS_ON_LIST(tp)) {
2194 tp->t_flags &= ~(TF_TIMER_ONLIST);
2195 LIST_REMOVE(&tp->tentry, le);
2196 listp->entries--;
2197
2198 tp->tentry.le.le_next = NULL;
2199 tp->tentry.le.le_prev = NULL;
2200 }
2201 continue;
2202 }
2203 active_count++;
2204
2205 /*
2206 * Store the next timerentry pointer before releasing the
2207 * list lock. If that entry has to be removed when we
2208 * release the lock, this pointer will be updated to the
2209 * element after that.
2210 */
2211 listp->next_te = next_te;
2212
2213 VERIFY_NEXT_LINK(&tp->tentry, le);
2214 VERIFY_PREV_LINK(&tp->tentry, le);
2215
2216 lck_mtx_unlock(&listp->mtx);
2217
2218 offset = tcp_run_conn_timer(tp, &te_mode,
2219 listp->probe_if_index);
2220
2221 lck_mtx_lock(&listp->mtx);
2222
2223 next_te = listp->next_te;
2224 listp->next_te = NULL;
2225
2226 if (offset > 0 && te_mode != 0) {
2227 list_mode |= te_mode;
2228
2229 if (next_timer == 0 || offset < next_timer) {
2230 next_timer = offset;
2231 }
2232 }
2233 }
2234
2235 if (!LIST_EMPTY(&listp->lhead)) {
2236 uint32_t next_mode = 0;
2237 if ((list_mode & TCP_TIMERLIST_10MS_MODE) ||
2238 (listp->pref_mode & TCP_TIMERLIST_10MS_MODE)) {
2239 next_mode = TCP_TIMERLIST_10MS_MODE;
2240 } else if ((list_mode & TCP_TIMERLIST_100MS_MODE) ||
2241 (listp->pref_mode & TCP_TIMERLIST_100MS_MODE)) {
2242 next_mode = TCP_TIMERLIST_100MS_MODE;
2243 } else {
2244 next_mode = TCP_TIMERLIST_500MS_MODE;
2245 }
2246
2247 if (next_mode != TCP_TIMERLIST_500MS_MODE) {
2248 listp->idleruns = 0;
2249 } else {
2250 /*
2251 * the next required mode is slow mode, but if
2252 * the last one was a faster mode and we did not
2253 * have enough idle runs, repeat the last mode.
2254 *
2255 * We try to keep the timer list in fast mode for
2256 * some idle time in expectation of new data.
2257 */
2258 if (listp->mode != next_mode &&
2259 listp->idleruns < timer_fastmode_idlemax) {
2260 listp->idleruns++;
2261 next_mode = listp->mode;
2262 next_timer = TCP_TIMER_100MS_QUANTUM;
2263 } else {
2264 listp->idleruns = 0;
2265 }
2266 }
2267 listp->mode = next_mode;
2268 if (listp->pref_offset != 0) {
2269 next_timer = min(listp->pref_offset, next_timer);
2270 }
2271
2272 if (listp->mode == TCP_TIMERLIST_500MS_MODE) {
2273 next_timer = max(next_timer,
2274 TCP_TIMER_500MS_QUANTUM);
2275 }
2276
2277 tcp_sched_timerlist(next_timer);
2278 } else {
2279 /*
2280 * No need to reschedule this timer, but always run
2281 * periodically at a much higher granularity.
2282 */
2283 tcp_sched_timerlist(TCP_TIMERLIST_MAX_OFFSET);
2284 }
2285
2286 listp->running = FALSE;
2287 listp->pref_mode = 0;
2288 listp->pref_offset = 0;
2289 listp->probe_if_index = 0;
2290
2291 lck_mtx_unlock(&listp->mtx);
2292 }
2293
2294 /*
2295 * Function to check if the timerlist needs to be rescheduled to run this
2296 * connection's timers correctly.
2297 */
2298 void
tcp_sched_timers(struct tcpcb * tp)2299 tcp_sched_timers(struct tcpcb *tp)
2300 {
2301 struct tcptimerentry *te = &tp->tentry;
2302 u_int16_t index = te->index;
2303 u_int16_t mode = te->mode;
2304 struct tcptimerlist *listp = &tcp_timer_list;
2305 int32_t offset = 0;
2306 boolean_t list_locked = FALSE;
2307
2308 if (tp->t_inpcb->inp_state == INPCB_STATE_DEAD) {
2309 /* Just return without adding the dead pcb to the list */
2310 if (TIMER_IS_ON_LIST(tp)) {
2311 tcp_remove_timer(tp);
2312 }
2313 return;
2314 }
2315
2316 if (index == TCPT_NONE) {
2317 /* Nothing to run */
2318 tcp_remove_timer(tp);
2319 return;
2320 }
2321
2322 /*
2323 * compute the offset at which the next timer for this connection
2324 * has to run.
2325 */
2326 offset = timer_diff(te->runtime, 0, tcp_now, 0);
2327 if (offset <= 0) {
2328 offset = 1;
2329 tcp_timer_advanced++;
2330 }
2331
2332 if (!TIMER_IS_ON_LIST(tp)) {
2333 if (!list_locked) {
2334 lck_mtx_lock(&listp->mtx);
2335 list_locked = TRUE;
2336 }
2337
2338 if (!TIMER_IS_ON_LIST(tp)) {
2339 LIST_INSERT_HEAD(&listp->lhead, te, le);
2340 tp->t_flags |= TF_TIMER_ONLIST;
2341
2342 listp->entries++;
2343 if (listp->entries > listp->maxentries) {
2344 listp->maxentries = listp->entries;
2345 }
2346
2347 /* if the list is not scheduled, just schedule it */
2348 if (!listp->scheduled) {
2349 goto schedule;
2350 }
2351 }
2352 }
2353
2354 /*
2355 * Timer entry is currently on the list, check if the list needs
2356 * to be rescheduled.
2357 */
2358 if (need_to_resched_timerlist(te->runtime, mode)) {
2359 tcp_resched_timerlist++;
2360
2361 if (!list_locked) {
2362 lck_mtx_lock(&listp->mtx);
2363 list_locked = TRUE;
2364 }
2365
2366 VERIFY_NEXT_LINK(te, le);
2367 VERIFY_PREV_LINK(te, le);
2368
2369 if (listp->running) {
2370 listp->pref_mode |= mode;
2371 if (listp->pref_offset == 0 ||
2372 offset < listp->pref_offset) {
2373 listp->pref_offset = offset;
2374 }
2375 } else {
2376 /*
2377 * The list could have got rescheduled while
2378 * this thread was waiting for the lock
2379 */
2380 if (listp->scheduled) {
2381 int32_t diff;
2382 diff = timer_diff(listp->runtime, 0,
2383 tcp_now, offset);
2384 if (diff <= 0) {
2385 goto done;
2386 } else {
2387 goto schedule;
2388 }
2389 } else {
2390 goto schedule;
2391 }
2392 }
2393 }
2394 goto done;
2395
2396 schedule:
2397 /*
2398 * Since a connection with timers is getting scheduled, the timer
2399 * list moves from idle to active state and that is why idlegen is
2400 * reset
2401 */
2402 if (mode & TCP_TIMERLIST_10MS_MODE) {
2403 listp->mode = TCP_TIMERLIST_10MS_MODE;
2404 listp->idleruns = 0;
2405 offset = min(offset, TCP_TIMER_10MS_QUANTUM);
2406 } else if (mode & TCP_TIMERLIST_100MS_MODE) {
2407 if (listp->mode > TCP_TIMERLIST_100MS_MODE) {
2408 listp->mode = TCP_TIMERLIST_100MS_MODE;
2409 }
2410 listp->idleruns = 0;
2411 offset = min(offset, TCP_TIMER_100MS_QUANTUM);
2412 }
2413 tcp_sched_timerlist(offset);
2414
2415 done:
2416 if (list_locked) {
2417 lck_mtx_unlock(&listp->mtx);
2418 }
2419
2420 return;
2421 }
2422
2423 static inline void
tcp_set_lotimer_index(struct tcpcb * tp)2424 tcp_set_lotimer_index(struct tcpcb *tp)
2425 {
2426 uint16_t i, lo_index = TCPT_NONE, mode = 0;
2427 uint32_t lo_timer = 0;
2428 for (i = 0; i < TCPT_NTIMERS; ++i) {
2429 if (tp->t_timer[i] != 0) {
2430 TCP_SET_TIMER_MODE(mode, i);
2431 if (lo_timer == 0 || tp->t_timer[i] < lo_timer) {
2432 lo_timer = tp->t_timer[i];
2433 lo_index = i;
2434 }
2435 }
2436 }
2437 tp->tentry.index = lo_index;
2438 tp->tentry.mode = mode;
2439 VERIFY(tp->tentry.index == TCPT_NONE || tp->tentry.mode > 0);
2440
2441 if (tp->tentry.index != TCPT_NONE) {
2442 tp->tentry.runtime = tp->tentry.timer_start
2443 + tp->t_timer[tp->tentry.index];
2444 if (tp->tentry.runtime == 0) {
2445 tp->tentry.runtime++;
2446 }
2447 }
2448 }
2449
2450 void
tcp_check_timer_state(struct tcpcb * tp)2451 tcp_check_timer_state(struct tcpcb *tp)
2452 {
2453 socket_lock_assert_owned(tp->t_inpcb->inp_socket);
2454
2455 if (tp->t_inpcb->inp_flags2 & INP2_TIMEWAIT) {
2456 return;
2457 }
2458
2459 tcp_set_lotimer_index(tp);
2460
2461 tcp_sched_timers(tp);
2462 return;
2463 }
2464
2465 static inline void
tcp_cumulative_stat(u_int32_t cur,u_int32_t * prev,u_int32_t * dest)2466 tcp_cumulative_stat(u_int32_t cur, u_int32_t *prev, u_int32_t *dest)
2467 {
2468 /* handle wrap around */
2469 int32_t diff = (int32_t) (cur - *prev);
2470 if (diff > 0) {
2471 *dest = diff;
2472 } else {
2473 *dest = 0;
2474 }
2475 *prev = cur;
2476 return;
2477 }
2478
2479 static inline void
tcp_cumulative_stat64(u_int64_t cur,u_int64_t * prev,u_int64_t * dest)2480 tcp_cumulative_stat64(u_int64_t cur, u_int64_t *prev, u_int64_t *dest)
2481 {
2482 /* handle wrap around */
2483 int64_t diff = (int64_t) (cur - *prev);
2484 if (diff > 0) {
2485 *dest = diff;
2486 } else {
2487 *dest = 0;
2488 }
2489 *prev = cur;
2490 return;
2491 }
2492
2493 __private_extern__ void
tcp_report_stats(void)2494 tcp_report_stats(void)
2495 {
2496 struct nstat_sysinfo_data data;
2497 struct sockaddr_in dst;
2498 struct sockaddr_in6 dst6;
2499 struct rtentry *rt = NULL;
2500 static struct tcp_last_report_stats prev;
2501 u_int64_t var, uptime;
2502
2503 #define stat data.u.tcp_stats
2504 if (((uptime = net_uptime()) - tcp_last_report_time) <
2505 tcp_report_stats_interval) {
2506 return;
2507 }
2508
2509 tcp_last_report_time = uptime;
2510
2511 bzero(&data, sizeof(data));
2512 data.flags = NSTAT_SYSINFO_TCP_STATS;
2513
2514 bzero(&dst, sizeof(dst));
2515 dst.sin_len = sizeof(dst);
2516 dst.sin_family = AF_INET;
2517
2518 /* ipv4 avg rtt */
2519 lck_mtx_lock(rnh_lock);
2520 rt = rt_lookup(TRUE, (struct sockaddr *)&dst, NULL,
2521 rt_tables[AF_INET], IFSCOPE_NONE);
2522 lck_mtx_unlock(rnh_lock);
2523 if (rt != NULL) {
2524 RT_LOCK(rt);
2525 if (rt_primary_default(rt, rt_key(rt)) &&
2526 rt->rt_stats != NULL) {
2527 stat.ipv4_avgrtt = rt->rt_stats->nstat_avg_rtt;
2528 }
2529 RT_UNLOCK(rt);
2530 rtfree(rt);
2531 rt = NULL;
2532 }
2533
2534 /* ipv6 avg rtt */
2535 bzero(&dst6, sizeof(dst6));
2536 dst6.sin6_len = sizeof(dst6);
2537 dst6.sin6_family = AF_INET6;
2538
2539 lck_mtx_lock(rnh_lock);
2540 rt = rt_lookup(TRUE, (struct sockaddr *)&dst6, NULL,
2541 rt_tables[AF_INET6], IFSCOPE_NONE);
2542 lck_mtx_unlock(rnh_lock);
2543 if (rt != NULL) {
2544 RT_LOCK(rt);
2545 if (rt_primary_default(rt, rt_key(rt)) &&
2546 rt->rt_stats != NULL) {
2547 stat.ipv6_avgrtt = rt->rt_stats->nstat_avg_rtt;
2548 }
2549 RT_UNLOCK(rt);
2550 rtfree(rt);
2551 rt = NULL;
2552 }
2553
2554 /* send packet loss rate, shift by 10 for precision */
2555 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_sndrexmitpack > 0) {
2556 var = tcpstat.tcps_sndrexmitpack << 10;
2557 stat.send_plr = (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2558 }
2559
2560 /* recv packet loss rate, shift by 10 for precision */
2561 if (tcpstat.tcps_rcvpack > 0 && tcpstat.tcps_recovered_pkts > 0) {
2562 var = tcpstat.tcps_recovered_pkts << 10;
2563 stat.recv_plr = (uint32_t)((var * 100) / tcpstat.tcps_rcvpack);
2564 }
2565
2566 /* RTO after tail loss, shift by 10 for precision */
2567 if (tcpstat.tcps_sndrexmitpack > 0
2568 && tcpstat.tcps_tailloss_rto > 0) {
2569 var = tcpstat.tcps_tailloss_rto << 10;
2570 stat.send_tlrto_rate =
2571 (uint32_t)((var * 100) / tcpstat.tcps_sndrexmitpack);
2572 }
2573
2574 /* packet reordering */
2575 if (tcpstat.tcps_sndpack > 0 && tcpstat.tcps_reordered_pkts > 0) {
2576 var = tcpstat.tcps_reordered_pkts << 10;
2577 stat.send_reorder_rate =
2578 (uint32_t)((var * 100) / tcpstat.tcps_sndpack);
2579 }
2580
2581 if (tcp_ecn_outbound == 1) {
2582 stat.ecn_client_enabled = 1;
2583 }
2584 if (tcp_ecn_inbound == 1) {
2585 stat.ecn_server_enabled = 1;
2586 }
2587 tcp_cumulative_stat(tcpstat.tcps_connattempt,
2588 &prev.tcps_connattempt, &stat.connection_attempts);
2589 tcp_cumulative_stat(tcpstat.tcps_accepts,
2590 &prev.tcps_accepts, &stat.connection_accepts);
2591 tcp_cumulative_stat(tcpstat.tcps_ecn_client_setup,
2592 &prev.tcps_ecn_client_setup, &stat.ecn_client_setup);
2593 tcp_cumulative_stat(tcpstat.tcps_ecn_server_setup,
2594 &prev.tcps_ecn_server_setup, &stat.ecn_server_setup);
2595 tcp_cumulative_stat(tcpstat.tcps_ecn_client_success,
2596 &prev.tcps_ecn_client_success, &stat.ecn_client_success);
2597 tcp_cumulative_stat(tcpstat.tcps_ecn_server_success,
2598 &prev.tcps_ecn_server_success, &stat.ecn_server_success);
2599 tcp_cumulative_stat(tcpstat.tcps_ecn_not_supported,
2600 &prev.tcps_ecn_not_supported, &stat.ecn_not_supported);
2601 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_syn,
2602 &prev.tcps_ecn_lost_syn, &stat.ecn_lost_syn);
2603 tcp_cumulative_stat(tcpstat.tcps_ecn_lost_synack,
2604 &prev.tcps_ecn_lost_synack, &stat.ecn_lost_synack);
2605 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ce,
2606 &prev.tcps_ecn_recv_ce, &stat.ecn_recv_ce);
2607 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2608 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2609 tcp_cumulative_stat(tcpstat.tcps_ecn_recv_ece,
2610 &prev.tcps_ecn_recv_ece, &stat.ecn_recv_ece);
2611 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2612 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2613 tcp_cumulative_stat(tcpstat.tcps_ecn_sent_ece,
2614 &prev.tcps_ecn_sent_ece, &stat.ecn_sent_ece);
2615 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ce,
2616 &prev.tcps_ecn_conn_recv_ce, &stat.ecn_conn_recv_ce);
2617 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_recv_ece,
2618 &prev.tcps_ecn_conn_recv_ece, &stat.ecn_conn_recv_ece);
2619 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_plnoce,
2620 &prev.tcps_ecn_conn_plnoce, &stat.ecn_conn_plnoce);
2621 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_pl_ce,
2622 &prev.tcps_ecn_conn_pl_ce, &stat.ecn_conn_pl_ce);
2623 tcp_cumulative_stat(tcpstat.tcps_ecn_conn_nopl_ce,
2624 &prev.tcps_ecn_conn_nopl_ce, &stat.ecn_conn_nopl_ce);
2625 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_synloss,
2626 &prev.tcps_ecn_fallback_synloss, &stat.ecn_fallback_synloss);
2627 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_reorder,
2628 &prev.tcps_ecn_fallback_reorder, &stat.ecn_fallback_reorder);
2629 tcp_cumulative_stat(tcpstat.tcps_ecn_fallback_ce,
2630 &prev.tcps_ecn_fallback_ce, &stat.ecn_fallback_ce);
2631 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_rcv,
2632 &prev.tcps_tfo_syn_data_rcv, &stat.tfo_syn_data_rcv);
2633 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req_rcv,
2634 &prev.tcps_tfo_cookie_req_rcv, &stat.tfo_cookie_req_rcv);
2635 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_sent,
2636 &prev.tcps_tfo_cookie_sent, &stat.tfo_cookie_sent);
2637 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_invalid,
2638 &prev.tcps_tfo_cookie_invalid, &stat.tfo_cookie_invalid);
2639 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_req,
2640 &prev.tcps_tfo_cookie_req, &stat.tfo_cookie_req);
2641 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_rcv,
2642 &prev.tcps_tfo_cookie_rcv, &stat.tfo_cookie_rcv);
2643 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_sent,
2644 &prev.tcps_tfo_syn_data_sent, &stat.tfo_syn_data_sent);
2645 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_data_acked,
2646 &prev.tcps_tfo_syn_data_acked, &stat.tfo_syn_data_acked);
2647 tcp_cumulative_stat(tcpstat.tcps_tfo_syn_loss,
2648 &prev.tcps_tfo_syn_loss, &stat.tfo_syn_loss);
2649 tcp_cumulative_stat(tcpstat.tcps_tfo_blackhole,
2650 &prev.tcps_tfo_blackhole, &stat.tfo_blackhole);
2651 tcp_cumulative_stat(tcpstat.tcps_tfo_cookie_wrong,
2652 &prev.tcps_tfo_cookie_wrong, &stat.tfo_cookie_wrong);
2653 tcp_cumulative_stat(tcpstat.tcps_tfo_no_cookie_rcv,
2654 &prev.tcps_tfo_no_cookie_rcv, &stat.tfo_no_cookie_rcv);
2655 tcp_cumulative_stat(tcpstat.tcps_tfo_heuristics_disable,
2656 &prev.tcps_tfo_heuristics_disable, &stat.tfo_heuristics_disable);
2657 tcp_cumulative_stat(tcpstat.tcps_tfo_sndblackhole,
2658 &prev.tcps_tfo_sndblackhole, &stat.tfo_sndblackhole);
2659
2660
2661 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_attempt,
2662 &prev.tcps_mptcp_handover_attempt, &stat.mptcp_handover_attempt);
2663 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_attempt,
2664 &prev.tcps_mptcp_interactive_attempt, &stat.mptcp_interactive_attempt);
2665 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_attempt,
2666 &prev.tcps_mptcp_aggregate_attempt, &stat.mptcp_aggregate_attempt);
2667 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_attempt,
2668 &prev.tcps_mptcp_fp_handover_attempt, &stat.mptcp_fp_handover_attempt);
2669 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_attempt,
2670 &prev.tcps_mptcp_fp_interactive_attempt, &stat.mptcp_fp_interactive_attempt);
2671 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_attempt,
2672 &prev.tcps_mptcp_fp_aggregate_attempt, &stat.mptcp_fp_aggregate_attempt);
2673 tcp_cumulative_stat(tcpstat.tcps_mptcp_heuristic_fallback,
2674 &prev.tcps_mptcp_heuristic_fallback, &stat.mptcp_heuristic_fallback);
2675 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_heuristic_fallback,
2676 &prev.tcps_mptcp_fp_heuristic_fallback, &stat.mptcp_fp_heuristic_fallback);
2677 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_wifi,
2678 &prev.tcps_mptcp_handover_success_wifi, &stat.mptcp_handover_success_wifi);
2679 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_success_cell,
2680 &prev.tcps_mptcp_handover_success_cell, &stat.mptcp_handover_success_cell);
2681 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_success,
2682 &prev.tcps_mptcp_interactive_success, &stat.mptcp_interactive_success);
2683 tcp_cumulative_stat(tcpstat.tcps_mptcp_aggregate_success,
2684 &prev.tcps_mptcp_aggregate_success, &stat.mptcp_aggregate_success);
2685 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_wifi,
2686 &prev.tcps_mptcp_fp_handover_success_wifi, &stat.mptcp_fp_handover_success_wifi);
2687 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_handover_success_cell,
2688 &prev.tcps_mptcp_fp_handover_success_cell, &stat.mptcp_fp_handover_success_cell);
2689 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_interactive_success,
2690 &prev.tcps_mptcp_fp_interactive_success, &stat.mptcp_fp_interactive_success);
2691 tcp_cumulative_stat(tcpstat.tcps_mptcp_fp_aggregate_success,
2692 &prev.tcps_mptcp_fp_aggregate_success, &stat.mptcp_fp_aggregate_success);
2693 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_cell_from_wifi,
2694 &prev.tcps_mptcp_handover_cell_from_wifi, &stat.mptcp_handover_cell_from_wifi);
2695 tcp_cumulative_stat(tcpstat.tcps_mptcp_handover_wifi_from_cell,
2696 &prev.tcps_mptcp_handover_wifi_from_cell, &stat.mptcp_handover_wifi_from_cell);
2697 tcp_cumulative_stat(tcpstat.tcps_mptcp_interactive_cell_from_wifi,
2698 &prev.tcps_mptcp_interactive_cell_from_wifi, &stat.mptcp_interactive_cell_from_wifi);
2699 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_cell_bytes,
2700 &prev.tcps_mptcp_handover_cell_bytes, &stat.mptcp_handover_cell_bytes);
2701 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_cell_bytes,
2702 &prev.tcps_mptcp_interactive_cell_bytes, &stat.mptcp_interactive_cell_bytes);
2703 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_cell_bytes,
2704 &prev.tcps_mptcp_aggregate_cell_bytes, &stat.mptcp_aggregate_cell_bytes);
2705 tcp_cumulative_stat64(tcpstat.tcps_mptcp_handover_all_bytes,
2706 &prev.tcps_mptcp_handover_all_bytes, &stat.mptcp_handover_all_bytes);
2707 tcp_cumulative_stat64(tcpstat.tcps_mptcp_interactive_all_bytes,
2708 &prev.tcps_mptcp_interactive_all_bytes, &stat.mptcp_interactive_all_bytes);
2709 tcp_cumulative_stat64(tcpstat.tcps_mptcp_aggregate_all_bytes,
2710 &prev.tcps_mptcp_aggregate_all_bytes, &stat.mptcp_aggregate_all_bytes);
2711 tcp_cumulative_stat(tcpstat.tcps_mptcp_back_to_wifi,
2712 &prev.tcps_mptcp_back_to_wifi, &stat.mptcp_back_to_wifi);
2713 tcp_cumulative_stat(tcpstat.tcps_mptcp_wifi_proxy,
2714 &prev.tcps_mptcp_wifi_proxy, &stat.mptcp_wifi_proxy);
2715 tcp_cumulative_stat(tcpstat.tcps_mptcp_cell_proxy,
2716 &prev.tcps_mptcp_cell_proxy, &stat.mptcp_cell_proxy);
2717 tcp_cumulative_stat(tcpstat.tcps_mptcp_triggered_cell,
2718 &prev.tcps_mptcp_triggered_cell, &stat.mptcp_triggered_cell);
2719
2720 nstat_sysinfo_send_data(&data);
2721
2722 #undef stat
2723 }
2724
2725 void
tcp_interface_send_probe(u_int16_t probe_if_index)2726 tcp_interface_send_probe(u_int16_t probe_if_index)
2727 {
2728 int32_t offset = 0;
2729 struct tcptimerlist *listp = &tcp_timer_list;
2730
2731 /* Make sure TCP clock is up to date */
2732 calculate_tcp_clock();
2733
2734 lck_mtx_lock(&listp->mtx);
2735 if (listp->probe_if_index > 0 && listp->probe_if_index != probe_if_index) {
2736 tcpstat.tcps_probe_if_conflict++;
2737 os_log(OS_LOG_DEFAULT,
2738 "%s: probe_if_index %u conflicts with %u, tcps_probe_if_conflict %u\n",
2739 __func__, probe_if_index, listp->probe_if_index,
2740 tcpstat.tcps_probe_if_conflict);
2741 goto done;
2742 }
2743
2744 listp->probe_if_index = probe_if_index;
2745 if (listp->running) {
2746 os_log(OS_LOG_DEFAULT, "%s: timer list already running for if_index %u\n",
2747 __func__, probe_if_index);
2748 goto done;
2749 }
2750
2751 /*
2752 * Reschedule the timerlist to run within the next 10ms, which is
2753 * the fastest that we can do.
2754 */
2755 offset = TCP_TIMER_10MS_QUANTUM;
2756 if (listp->scheduled) {
2757 int32_t diff;
2758 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2759 if (diff <= 0) {
2760 /* The timer will fire sooner than what's needed */
2761 os_log(OS_LOG_DEFAULT,
2762 "%s: timer will fire sooner than needed for if_index %u\n",
2763 __func__, probe_if_index);
2764 goto done;
2765 }
2766 }
2767 listp->mode = TCP_TIMERLIST_10MS_MODE;
2768 listp->idleruns = 0;
2769
2770 tcp_sched_timerlist(offset);
2771
2772 done:
2773 lck_mtx_unlock(&listp->mtx);
2774 return;
2775 }
2776
2777 /*
2778 * Enable read probes on this connection, if:
2779 * - it is in established state
2780 * - doesn't have any data outstanding
2781 * - the outgoing ifp matches
2782 * - we have not already sent any read probes
2783 */
2784 static void
tcp_enable_read_probe(struct tcpcb * tp,struct ifnet * ifp)2785 tcp_enable_read_probe(struct tcpcb *tp, struct ifnet *ifp)
2786 {
2787 if (tp->t_state == TCPS_ESTABLISHED &&
2788 tp->snd_max == tp->snd_una &&
2789 tp->t_inpcb->inp_last_outifp == ifp &&
2790 !(tp->t_flagsext & TF_DETECT_READSTALL) &&
2791 tp->t_rtimo_probes == 0) {
2792 tp->t_flagsext |= TF_DETECT_READSTALL;
2793 tp->t_rtimo_probes = 0;
2794 tp->t_timer[TCPT_KEEP] = OFFSET_FROM_START(tp,
2795 TCP_TIMER_10MS_QUANTUM);
2796 if (tp->tentry.index == TCPT_NONE) {
2797 tp->tentry.index = TCPT_KEEP;
2798 tp->tentry.runtime = tcp_now +
2799 TCP_TIMER_10MS_QUANTUM;
2800 } else {
2801 int32_t diff = 0;
2802
2803 /* Reset runtime to be in next 10ms */
2804 diff = timer_diff(tp->tentry.runtime, 0,
2805 tcp_now, TCP_TIMER_10MS_QUANTUM);
2806 if (diff > 0) {
2807 tp->tentry.index = TCPT_KEEP;
2808 tp->tentry.runtime = tcp_now +
2809 TCP_TIMER_10MS_QUANTUM;
2810 if (tp->tentry.runtime == 0) {
2811 tp->tentry.runtime++;
2812 }
2813 }
2814 }
2815 }
2816 }
2817
2818 /*
2819 * Disable read probe and reset the keep alive timer
2820 */
2821 static void
tcp_disable_read_probe(struct tcpcb * tp)2822 tcp_disable_read_probe(struct tcpcb *tp)
2823 {
2824 if (tp->t_adaptive_rtimo == 0 &&
2825 ((tp->t_flagsext & TF_DETECT_READSTALL) ||
2826 tp->t_rtimo_probes > 0)) {
2827 tcp_keepalive_reset(tp);
2828
2829 if (tp->t_mpsub) {
2830 mptcp_reset_keepalive(tp);
2831 }
2832 }
2833 }
2834
2835 /*
2836 * Reschedule the tcp timerlist in the next 10ms to re-enable read/write
2837 * probes on connections going over a particular interface.
2838 */
2839 void
tcp_probe_connectivity(struct ifnet * ifp,u_int32_t enable)2840 tcp_probe_connectivity(struct ifnet *ifp, u_int32_t enable)
2841 {
2842 int32_t offset;
2843 struct tcptimerlist *listp = &tcp_timer_list;
2844 struct inpcbinfo *pcbinfo = &tcbinfo;
2845 struct inpcb *inp, *nxt;
2846
2847 if (ifp == NULL) {
2848 return;
2849 }
2850
2851 /* update clock */
2852 calculate_tcp_clock();
2853
2854 /*
2855 * Enable keep alive timer on all connections that are
2856 * active/established on this interface.
2857 */
2858 lck_rw_lock_shared(&pcbinfo->ipi_lock);
2859
2860 LIST_FOREACH_SAFE(inp, pcbinfo->ipi_listhead, inp_list, nxt) {
2861 struct tcpcb *tp = NULL;
2862 if (in_pcb_checkstate(inp, WNT_ACQUIRE, 0) ==
2863 WNT_STOPUSING) {
2864 continue;
2865 }
2866
2867 /* Acquire lock to look at the state of the connection */
2868 socket_lock(inp->inp_socket, 1);
2869
2870 /* Release the want count */
2871 if (inp->inp_ppcb == NULL ||
2872 (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING)) {
2873 socket_unlock(inp->inp_socket, 1);
2874 continue;
2875 }
2876 tp = intotcpcb(inp);
2877 if (enable) {
2878 tcp_enable_read_probe(tp, ifp);
2879 } else {
2880 tcp_disable_read_probe(tp);
2881 }
2882
2883 socket_unlock(inp->inp_socket, 1);
2884 }
2885 lck_rw_done(&pcbinfo->ipi_lock);
2886
2887 lck_mtx_lock(&listp->mtx);
2888 if (listp->running) {
2889 listp->pref_mode |= TCP_TIMERLIST_10MS_MODE;
2890 goto done;
2891 }
2892
2893 /* Reschedule within the next 10ms */
2894 offset = TCP_TIMER_10MS_QUANTUM;
2895 if (listp->scheduled) {
2896 int32_t diff;
2897 diff = timer_diff(listp->runtime, 0, tcp_now, offset);
2898 if (diff <= 0) {
2899 /* The timer will fire sooner than what's needed */
2900 goto done;
2901 }
2902 }
2903 listp->mode = TCP_TIMERLIST_10MS_MODE;
2904 listp->idleruns = 0;
2905
2906 tcp_sched_timerlist(offset);
2907 done:
2908 lck_mtx_unlock(&listp->mtx);
2909 return;
2910 }
2911
2912 inline void
tcp_update_mss_core(struct tcpcb * tp,struct ifnet * ifp)2913 tcp_update_mss_core(struct tcpcb *tp, struct ifnet *ifp)
2914 {
2915 struct if_cellular_status_v1 *ifsr;
2916 u_int32_t optlen;
2917 ifsr = &ifp->if_link_status->ifsr_u.ifsr_cell.if_cell_u.if_status_v1;
2918 if (ifsr->valid_bitmask & IF_CELL_UL_MSS_RECOMMENDED_VALID) {
2919 optlen = tp->t_maxopd - tp->t_maxseg;
2920
2921 if (ifsr->mss_recommended ==
2922 IF_CELL_UL_MSS_RECOMMENDED_NONE &&
2923 tp->t_cached_maxopd > 0 &&
2924 tp->t_maxopd < tp->t_cached_maxopd) {
2925 tp->t_maxopd = tp->t_cached_maxopd;
2926 tcpstat.tcps_mss_to_default++;
2927 } else if (ifsr->mss_recommended ==
2928 IF_CELL_UL_MSS_RECOMMENDED_MEDIUM &&
2929 tp->t_maxopd > tcp_mss_rec_medium) {
2930 tp->t_cached_maxopd = tp->t_maxopd;
2931 tp->t_maxopd = tcp_mss_rec_medium;
2932 tcpstat.tcps_mss_to_medium++;
2933 } else if (ifsr->mss_recommended ==
2934 IF_CELL_UL_MSS_RECOMMENDED_LOW &&
2935 tp->t_maxopd > tcp_mss_rec_low) {
2936 tp->t_cached_maxopd = tp->t_maxopd;
2937 tp->t_maxopd = tcp_mss_rec_low;
2938 tcpstat.tcps_mss_to_low++;
2939 }
2940 tp->t_maxseg = tp->t_maxopd - optlen;
2941
2942 /*
2943 * clear the cached value if it is same as the current
2944 */
2945 if (tp->t_maxopd == tp->t_cached_maxopd) {
2946 tp->t_cached_maxopd = 0;
2947 }
2948 }
2949 }
2950
2951 void
tcp_update_mss_locked(struct socket * so,struct ifnet * ifp)2952 tcp_update_mss_locked(struct socket *so, struct ifnet *ifp)
2953 {
2954 struct inpcb *inp = sotoinpcb(so);
2955 struct tcpcb *tp = intotcpcb(inp);
2956
2957 if (ifp == NULL && (ifp = inp->inp_last_outifp) == NULL) {
2958 return;
2959 }
2960
2961 if (!IFNET_IS_CELLULAR(ifp)) {
2962 /*
2963 * This optimization is implemented for cellular
2964 * networks only
2965 */
2966 return;
2967 }
2968 if (tp->t_state <= TCPS_CLOSE_WAIT) {
2969 /*
2970 * If the connection is currently doing or has done PMTU
2971 * blackhole detection, do not change the MSS
2972 */
2973 if (tp->t_flags & TF_BLACKHOLE) {
2974 return;
2975 }
2976 if (ifp->if_link_status == NULL) {
2977 return;
2978 }
2979 tcp_update_mss_core(tp, ifp);
2980 }
2981 }
2982
2983 void
tcp_itimer(struct inpcbinfo * ipi)2984 tcp_itimer(struct inpcbinfo *ipi)
2985 {
2986 struct inpcb *inp, *nxt;
2987
2988 if (lck_rw_try_lock_exclusive(&ipi->ipi_lock) == FALSE) {
2989 if (tcp_itimer_done == TRUE) {
2990 tcp_itimer_done = FALSE;
2991 atomic_add_32(&ipi->ipi_timer_req.intimer_fast, 1);
2992 return;
2993 }
2994 /* Upgrade failed, lost lock now take it again exclusive */
2995 lck_rw_lock_exclusive(&ipi->ipi_lock);
2996 }
2997 tcp_itimer_done = TRUE;
2998
2999 LIST_FOREACH_SAFE(inp, &tcb, inp_list, nxt) {
3000 struct socket *so;
3001 struct ifnet *ifp;
3002
3003 if (inp->inp_ppcb == NULL ||
3004 in_pcb_checkstate(inp, WNT_ACQUIRE, 0) == WNT_STOPUSING) {
3005 continue;
3006 }
3007 so = inp->inp_socket;
3008 ifp = inp->inp_last_outifp;
3009 socket_lock(so, 1);
3010 if (in_pcb_checkstate(inp, WNT_RELEASE, 1) == WNT_STOPUSING) {
3011 socket_unlock(so, 1);
3012 continue;
3013 }
3014 so_check_extended_bk_idle_time(so);
3015 if (ipi->ipi_flags & INPCBINFO_UPDATE_MSS) {
3016 tcp_update_mss_locked(so, NULL);
3017 }
3018 socket_unlock(so, 1);
3019
3020 /*
3021 * Defunct all system-initiated background sockets if the
3022 * socket is using the cellular interface and the interface
3023 * has its LQM set to abort.
3024 */
3025 if ((ipi->ipi_flags & INPCBINFO_HANDLE_LQM_ABORT) &&
3026 IS_SO_TC_BACKGROUNDSYSTEM(so->so_traffic_class) &&
3027 ifp != NULL && IFNET_IS_CELLULAR(ifp) &&
3028 (ifp->if_interface_state.valid_bitmask &
3029 IF_INTERFACE_STATE_LQM_STATE_VALID) &&
3030 ifp->if_interface_state.lqm_state ==
3031 IFNET_LQM_THRESH_ABORT) {
3032 socket_defunct(current_proc(), so,
3033 SHUTDOWN_SOCKET_LEVEL_DISCONNECT_ALL);
3034 }
3035 }
3036
3037 ipi->ipi_flags &= ~(INPCBINFO_UPDATE_MSS | INPCBINFO_HANDLE_LQM_ABORT);
3038 lck_rw_done(&ipi->ipi_lock);
3039 }
3040