1 /*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /* TCP-pacing implementation and helper functions */
30
31 #include "tcp_includes.h"
32
33 static uint64_t
microuptime_ns(void)34 microuptime_ns(void)
35 {
36 uint64_t abstime = mach_absolute_time();
37 uint64_t ns = 0;
38
39 absolutetime_to_nanoseconds(abstime, &ns);
40
41 return ns;
42 }
43
44 /* Compute interval to use for specified (size) amount of data */
45 static uint32_t
tcp_pacer_get_packet_interval(struct tcpcb * tp,uint64_t size)46 tcp_pacer_get_packet_interval(struct tcpcb *tp, uint64_t size)
47 {
48 uint64_t rate = tp->t_pacer.rate;
49 uint64_t interval;
50
51 if (rate == 0) {
52 os_log_error(OS_LOG_DEFAULT,
53 "%s: pacer rate shouldn't be 0, CCA is %s (cwnd=%u, smoothed rtt=%u ms)",
54 __func__, CC_ALGO(tp)->name, tp->snd_cwnd, tp->t_srtt >> TCP_RTT_SHIFT);
55
56 return 0;
57 }
58
59 interval = (size * NSEC_PER_SEC) / rate;
60
61 if (interval > UINT32_MAX) {
62 interval = UINT32_MAX;
63 }
64
65 return (uint32_t)interval;
66 }
67
68 /*
69 * Computes packet's (of length pkt_len) tx_time according to the TCP-connection
70 * state. Also, returns the delay between now and the tx_time in milli-seconds.
71 * All values are in nano-seconds.
72 */
73 uint32_t
tcp_pacer_get_packet_tx_time(struct tcpcb * tp,int pkt_len,uint64_t * tx_time)74 tcp_pacer_get_packet_tx_time(struct tcpcb *tp, int pkt_len, uint64_t *tx_time)
75 {
76 uint64_t now = microuptime_ns();
77
78 if (pkt_len < 0) {
79 pkt_len = 0;
80 }
81
82 if (tp->t_pacer.packet_tx_time == 0) {
83 tp->t_pacer.packet_tx_time = now;
84 tp->t_pacer.current_size = pkt_len;
85 } else {
86 if (tp->t_pacer.current_size >= tp->t_pacer.tso_burst_size) {
87 /*
88 * Increment tx_time by packet_interval and
89 * reset current_size to this packet's len
90 */
91 tp->t_pacer.packet_tx_time +=
92 tcp_pacer_get_packet_interval(tp, tp->t_pacer.current_size);
93 tp->t_pacer.current_size = pkt_len;
94 if (now > tp->t_pacer.packet_tx_time) {
95 /*
96 * If current time is bigger, then application
97 * has already paced the packet. Also, we can't
98 * set tx_time in the past.
99 */
100 tp->t_pacer.packet_tx_time = now;
101 }
102 } else {
103 tp->t_pacer.current_size += pkt_len;
104 }
105 }
106
107 if (now < tp->t_pacer.packet_tx_time) {
108 *tx_time = tp->t_pacer.packet_tx_time;
109 } else {
110 *tx_time = now;
111 }
112
113 /*
114 * tcp_pacer_get_packet_interval() guarantees that the below substraction
115 * is less than UINT32_MAX.
116 */
117 return (uint32_t)(*tx_time - now) / NSEC_PER_MSEC;
118 }
119
120 #define MSEC_PER_SEC (1000) /* milliseconds per second */
121 uint64_t
tcp_compute_measured_rate(const struct tcpcb * tp)122 tcp_compute_measured_rate(const struct tcpcb *tp)
123 {
124 uint32_t srtt = tp->t_srtt;
125 uint64_t rate;
126
127 if (srtt == 0) {
128 /* Can't pace when it's at 0 */
129 return 0;
130 }
131
132 rate = tp->snd_cwnd;
133
134 /* Multiply by MSEC_PER_SEC as srtt is in milliseconds */
135 rate *= MSEC_PER_SEC;
136 rate = (rate << TCP_RTT_SHIFT) / srtt;
137
138 return rate;
139 }
140
141 #define BURST_SHIFT (12) /* 1/(2^12) = 0.000244s, we allow a burst queue of at least 250us */
142 void
tcp_update_pacer_state(struct tcpcb * tp)143 tcp_update_pacer_state(struct tcpcb *tp)
144 {
145 struct inpcb *inp = tp->t_inpcb;
146 uint32_t burst;
147 uint64_t rate;
148
149 rate = tcp_compute_measured_rate(tp);
150 /* Use 200% rate when in slow start */
151 if (tp->snd_cwnd < tp->snd_ssthresh) {
152 rate *= 2;
153 }
154
155 if (inp->inp_max_pacing_rate != UINT64_MAX) {
156 if (inp->inp_max_pacing_rate < rate) {
157 rate = inp->inp_max_pacing_rate;
158 }
159 }
160 burst = (uint32_t)(rate >> BURST_SHIFT);
161
162 tp->t_pacer.rate = rate;
163 tp->t_pacer.tso_burst_size = max(tp->t_maxseg, burst);
164 }
165