1 /*
2 * Copyright (c) 2013-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include "tcp_includes.h"
30
31 #include <sys/param.h>
32 #include <sys/kernel.h>
33 #include <sys/syslog.h>
34 #include <sys/kern_control.h>
35 #include <sys/domain.h>
36
37 #include <netinet/in.h>
38 #include <mach/sdt.h>
39 #include <libkern/OSAtomic.h>
40
41 #include <libkern/OSTypes.h>
42
43 extern struct tcp_cc_algo tcp_cc_newreno;
44 extern struct tcp_cc_algo tcp_cc_ledbat;
45 extern struct tcp_cc_algo tcp_cc_cubic;
46 extern struct tcp_cc_algo tcp_cc_prague;
47
48 #define SET_SNDSB_IDEAL_SIZE(sndsb, size) \
49 sndsb->sb_idealsize = min(max(tcp_sendspace, tp->snd_ssthresh), \
50 tcp_autosndbuf_max);
51
52 /* Array containing pointers to currently implemented TCP CC algorithms */
53 struct tcp_cc_algo* tcp_cc_algo_list[TCP_CC_ALGO_COUNT];
54
55 static struct tcp_cc_algo tcp_cc_algo_none;
56 /*
57 * Initialize TCP congestion control algorithms.
58 */
59
60 void
tcp_cc_init(void)61 tcp_cc_init(void)
62 {
63 bzero(&tcp_cc_algo_list, sizeof(tcp_cc_algo_list));
64 bzero(&tcp_cc_algo_none, sizeof(tcp_cc_algo_none));
65
66 tcp_cc_algo_list[TCP_CC_ALGO_NONE] = &tcp_cc_algo_none;
67 tcp_cc_algo_list[TCP_CC_ALGO_NEWRENO_INDEX] = &tcp_cc_newreno;
68 tcp_cc_algo_list[TCP_CC_ALGO_BACKGROUND_INDEX] = &tcp_cc_ledbat;
69 tcp_cc_algo_list[TCP_CC_ALGO_CUBIC_INDEX] = &tcp_cc_cubic;
70 tcp_cc_algo_list[TCP_CC_ALGO_PRAGUE_INDEX] = &tcp_cc_prague;
71
72 tcp_ccdbg_control_register();
73 }
74
75 void
tcp_cc_resize_sndbuf(struct tcpcb * tp)76 tcp_cc_resize_sndbuf(struct tcpcb *tp)
77 {
78 struct sockbuf *sb;
79 /*
80 * If the send socket buffer size is bigger than ssthresh,
81 * it is time to trim it because we do not want to hold
82 * too many mbufs in the socket buffer
83 */
84 sb = &tp->t_inpcb->inp_socket->so_snd;
85 if (sb->sb_hiwat > tp->snd_ssthresh &&
86 (sb->sb_flags & SB_AUTOSIZE)) {
87 if (sb->sb_idealsize > tp->snd_ssthresh) {
88 SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh);
89 }
90 sb->sb_flags |= SB_TRIM;
91 }
92 }
93
94 void
tcp_bad_rexmt_fix_sndbuf(struct tcpcb * tp)95 tcp_bad_rexmt_fix_sndbuf(struct tcpcb *tp)
96 {
97 struct sockbuf *sb;
98 sb = &tp->t_inpcb->inp_socket->so_snd;
99 if ((sb->sb_flags & (SB_TRIM | SB_AUTOSIZE)) == (SB_TRIM | SB_AUTOSIZE)) {
100 /*
101 * If there was a retransmission that was not necessary
102 * then the size of socket buffer can be restored to
103 * what it was before
104 */
105 SET_SNDSB_IDEAL_SIZE(sb, tp->snd_ssthresh);
106 if (sb->sb_hiwat <= sb->sb_idealsize) {
107 sbreserve(sb, sb->sb_idealsize);
108 sb->sb_flags &= ~SB_TRIM;
109 }
110 }
111 }
112
113 /*
114 * Calculate initial cwnd according to RFC3390.
115 */
116 void
tcp_cc_cwnd_init_or_reset(struct tcpcb * tp)117 tcp_cc_cwnd_init_or_reset(struct tcpcb *tp)
118 {
119 if (tp->t_flags & TF_LOCAL) {
120 tp->snd_cwnd = tp->t_maxseg * ss_fltsz_local;
121 } else {
122 if (tcp_cubic_minor_fixes) {
123 tp->snd_cwnd = tcp_initial_cwnd(tp);
124 } else {
125 /* initial congestion window according to RFC 3390 */
126 tp->snd_cwnd = min(4 * tp->t_maxseg,
127 max(2 * tp->t_maxseg, TCP_CC_CWND_INIT_BYTES));
128 }
129 }
130 }
131
132 /*
133 * Indicate whether this ack should be delayed.
134 * Here is the explanation for different settings of tcp_delack_enabled:
135 * - when set to 1, the behavior is same as when set to 2. We kept this
136 * for binary compatibility.
137 * - when set to 2, will "ack every other packet"
138 * - if our last ack wasn't a 0-sized window.
139 * - if the peer hasn't sent us a TH_PUSH data packet (radar 3649245).
140 * If TH_PUSH is set, take this as a clue that we need to ACK
141 * with no delay. This helps higher level protocols who
142 * won't send us more data even if the window is open
143 * because their last "segment" hasn't been ACKed
144 * - when set to 3, will do "streaming detection"
145 * - if we receive more than "maxseg_unacked" full packets
146 * in the last 100ms
147 * - if the connection is not in slow-start or idle or
148 * loss/recovery states
149 * - if those criteria aren't met, it will ack every other packet.
150 */
151 int
tcp_cc_delay_ack(struct tcpcb * tp,struct tcphdr * th)152 tcp_cc_delay_ack(struct tcpcb *tp, struct tcphdr *th)
153 {
154 switch (tcp_delack_enabled) {
155 case 1:
156 case 2:
157 if ((tp->t_flags & TF_RXWIN0SENT) == 0 &&
158 (th->th_flags & TH_PUSH) == 0 &&
159 (tp->t_unacksegs == 1)) {
160 return 1;
161 }
162 break;
163 case 3:
164 if (tcp_ack_strategy == TCP_ACK_STRATEGY_LEGACY) {
165 if ((tp->t_flags & TF_RXWIN0SENT) == 0 &&
166 (th->th_flags & TH_PUSH) == 0 &&
167 ((tp->t_unacksegs == 1) ||
168 ((tp->t_flags & TF_STRETCHACK) &&
169 tp->t_unacksegs < maxseg_unacked))) {
170 return 1;
171 }
172 } else {
173 uint32_t recwin;
174
175 /* Get the receive-window we would announce */
176 recwin = tcp_sbspace(tp);
177 if (recwin > (uint32_t)(TCP_MAXWIN << tp->rcv_scale)) {
178 recwin = (uint32_t)(TCP_MAXWIN << tp->rcv_scale);
179 }
180
181 /* Delay ACK, if:
182 *
183 * 1. We are not sending a zero-window
184 * 2. We are not forcing fast ACKs
185 * 3. We have more than the low-water mark in receive-buffer
186 * 4. The receive-window is not increasing
187 * 5. We have less than or equal of an MSS unacked or
188 * Window actually has been growing larger than the initial value by half of it.
189 * (this makes sure that during ramp-up we ACK every second MSS
190 * until we pass the tcp_recvspace * 1.5-threshold)
191 * 6. We haven't waited for half a BDP
192 * 7. The amount of unacked data is less than the maximum ACK-burst (256 MSS)
193 * We try to avoid having the sender end up hitting huge ACK-ranges.
194 *
195 * (a note on 6: The receive-window is
196 * roughly 2 BDP. Thus, recwin / 4 means half a BDP and
197 * thus we enforce an ACK roughly twice per RTT - even
198 * if the app does not read)
199 */
200 if ((tp->t_flags & TF_RXWIN0SENT) == 0 &&
201 tp->t_forced_acks == 0 &&
202 tp->t_inpcb->inp_socket->so_rcv.sb_cc > tp->t_inpcb->inp_socket->so_rcv.sb_lowat &&
203 recwin <= tp->t_last_recwin &&
204 (tp->rcv_nxt - tp->last_ack_sent <= tp->t_maxseg ||
205 recwin > (uint32_t)(tcp_recvspace + (tcp_recvspace >> 1))) &&
206 (tp->rcv_nxt - tp->last_ack_sent) < (recwin >> 2) &&
207 (tp->rcv_nxt - tp->last_ack_sent) < 256 * tp->t_maxseg) {
208 tp->t_stat.acks_delayed++;
209 return 1;
210 }
211 }
212 break;
213 }
214 return 0;
215 }
216
217 void
tcp_cc_allocate_state(struct tcpcb * tp)218 tcp_cc_allocate_state(struct tcpcb *tp)
219 {
220 if ((tp->tcp_cc_index == TCP_CC_ALGO_CUBIC_INDEX ||
221 tp->tcp_cc_index == TCP_CC_ALGO_PRAGUE_INDEX ||
222 tp->tcp_cc_index == TCP_CC_ALGO_BACKGROUND_INDEX) &&
223 tp->t_ccstate == NULL) {
224 tp->t_ccstate = &tp->_t_ccstate;
225
226 bzero(tp->t_ccstate, sizeof(*tp->t_ccstate));
227 }
228 }
229
230 /*
231 * If stretch ack was disabled automatically on long standing connections,
232 * re-evaluate the situation after 15 minutes to enable it.
233 */
234 #define TCP_STRETCHACK_DISABLE_WIN (15 * 60 * TCP_RETRANSHZ)
235 void
tcp_cc_after_idle_stretchack(struct tcpcb * tp)236 tcp_cc_after_idle_stretchack(struct tcpcb *tp)
237 {
238 struct tcp_globals *globals;
239 int32_t tdiff;
240
241 if (!(tp->t_flagsext & TF_DISABLE_STRETCHACK)) {
242 return;
243 }
244
245 globals = tcp_get_globals(tp);
246 tdiff = timer_diff(tcp_globals_now(globals), 0, tp->rcv_nostrack_ts, 0);
247 if (tdiff < 0) {
248 tdiff = -tdiff;
249 }
250
251 if (tdiff > TCP_STRETCHACK_DISABLE_WIN) {
252 tp->t_flagsext &= ~TF_DISABLE_STRETCHACK;
253 tp->t_stretchack_delayed = 0;
254
255 tcp_reset_stretch_ack(tp);
256 }
257 }
258
259 /*
260 * Detect if the congestion window is non-validated according to
261 * draft-ietf-tcpm-newcwv-07
262 */
263 inline uint32_t
tcp_cc_is_cwnd_nonvalidated(struct tcpcb * tp)264 tcp_cc_is_cwnd_nonvalidated(struct tcpcb *tp)
265 {
266 struct socket *so = tp->t_inpcb->inp_socket;
267
268 if (tp->t_pipeack == 0) {
269 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
270 return 0;
271 }
272
273 /*
274 * The congestion window is validated if the number of bytes acked
275 * is more than half of the current window or if there is more
276 * data to send in the send socket buffer
277 */
278 if (tp->t_pipeack >= (tp->snd_cwnd >> 1) ||
279 (so != NULL && so->so_snd.sb_cc > tp->snd_cwnd)) {
280 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
281 } else {
282 tp->t_flagsext |= TF_CWND_NONVALIDATED;
283 }
284
285 return tp->t_flagsext & TF_CWND_NONVALIDATED;
286 }
287
288 /*
289 * Adjust congestion window in response to congestion in non-validated
290 * phase.
291 */
292 inline void
tcp_cc_adjust_nonvalidated_cwnd(struct tcpcb * tp)293 tcp_cc_adjust_nonvalidated_cwnd(struct tcpcb *tp)
294 {
295 tp->t_pipeack = tcp_get_max_pipeack(tp);
296 tcp_clear_pipeack_state(tp);
297 tp->snd_cwnd = (max(tp->t_pipeack, tp->t_lossflightsize) >> 1);
298 if (tcp_cubic_minor_fixes) {
299 tp->snd_cwnd = max(tp->snd_cwnd, tp->t_maxseg);
300 } else {
301 tp->snd_cwnd = max(tp->snd_cwnd, TCP_CC_CWND_INIT_BYTES);
302 }
303 tp->snd_cwnd += tp->t_maxseg * tcprexmtthresh;
304 tp->t_flagsext &= ~TF_CWND_NONVALIDATED;
305 }
306
307 /*
308 * Return maximum of all the pipeack samples. Since the number of samples
309 * TCP_PIPEACK_SAMPLE_COUNT is 3 at this time, it will be simpler to do
310 * a comparision. We should change ths if the number of samples increases.
311 */
312 inline uint32_t
tcp_get_max_pipeack(struct tcpcb * tp)313 tcp_get_max_pipeack(struct tcpcb *tp)
314 {
315 uint32_t max_pipeack = 0;
316 max_pipeack = (tp->t_pipeack_sample[0] > tp->t_pipeack_sample[1]) ?
317 tp->t_pipeack_sample[0] : tp->t_pipeack_sample[1];
318 max_pipeack = (tp->t_pipeack_sample[2] > max_pipeack) ?
319 tp->t_pipeack_sample[2] : max_pipeack;
320
321 return max_pipeack;
322 }
323
324 inline void
tcp_clear_pipeack_state(struct tcpcb * tp)325 tcp_clear_pipeack_state(struct tcpcb *tp)
326 {
327 bzero(tp->t_pipeack_sample, sizeof(tp->t_pipeack_sample));
328 tp->t_pipeack_ind = 0;
329 tp->t_lossflightsize = 0;
330 }
331