1 /*
2 * Copyright (c) 2012-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 /*
30 * Flow Control and Feedback Advisory
31 *
32 * Each mbuf that is being sent out through an interface is tagged with a
33 * unique 32-bit ID which will help to identify all the packets that belong
34 * to a particular flow at the interface layer. Packets carrying such ID
35 * would need to be marked with PKTF_FLOW_ID. Normally, this ID is computed
36 * by the module that generates the flow. There are 3 kinds of flow sources
37 * that are currently recognized:
38 *
39 * a. INPCB (INET/INET6 Protocol Control Block). When a socket is
40 * connected, the flow hash for the socket is computed and stored in
41 * the PCB. Further transmissions on the socket will cause the hash
42 * value to be carried within the mbuf as the flow ID.
43 *
44 * b. Interface. When an interface is attached, the flow hash for the
45 * interface is computed and stored in the ifnet. This value is
46 * normally ignored for most network drivers, except for those that
47 * reside atop another driver, e.g. a virtual interface performing
48 * encapsulation/encryption on the original packet and sending the
49 * newly-generated packet to another interface. Such interface needs
50 * to associate all generated packets with the interface flow hash
51 * value as the flow ID.
52 *
53 * c. PF (Packet Filter). When a packet goes through PF and it is not
54 * already associated with a flow ID, PF will compute a flow hash and
55 * store it in the packet as flow ID. When the packet is associated
56 * with a PF state, the state record will have the flow ID stored
57 * within, in order to avoid recalculating the flow hash. Although PF
58 * is capable of generating flow IDs, it does not participate in flow
59 * advisory, and therefore packets whose IDs are computed by PF will
60 * not have their PKTF_FLOW_ADV packet flag set.
61 *
62 * Activation of flow advisory mechanism is done by setting the PKTF_FLOW_ADV
63 * packet flag; because a flow ID is required, the mechanism will not take
64 * place unless PKTF_FLOW_ID is set as well. The packet must also carry one
65 * of the flow source types FLOWSRC_{INPCB,IFNET} in order to identify where
66 * the flow advisory notification should be delivered to. As noted above,
67 * FLOWSRC_PF does not participate in this mechanism.
68 *
69 * The classq module configured on the interface is responsible for exerting
70 * flow control to the upper layers. This occurs when the number of packets
71 * queued for a flow reaches a limit. The module generating the flow will
72 * cease transmission until further flow advisory notice, and the flow will
73 * be inserted into the classq's flow control list.
74 *
75 * When packets are dequeued from the classq and the number of packets for
76 * a flow goes below a limit, the classq will transfer its flow control list
77 * to the global fadv_list. This will then trigger the flow advisory thread
78 * to run, which will cause the flow source modules to be notified that data
79 * can now be generated for those previously flow-controlled flows.
80 */
81
82 #include <sys/param.h>
83 #include <sys/systm.h>
84 #include <sys/kernel.h>
85 #include <sys/mcache.h>
86 #include <sys/mbuf.h>
87 #include <sys/proc_internal.h>
88 #include <sys/socketvar.h>
89
90 #include <kern/assert.h>
91 #include <kern/thread.h>
92 #include <kern/locks.h>
93 #include <kern/zalloc.h>
94
95 #include <netinet/in_pcb.h>
96 #include <net/flowadv.h>
97 #if SKYWALK
98 #include <skywalk/os_channel.h>
99 #endif /* SKYWALK */
100
101 /* Lock group and attribute for fadv_lock */
102 static LCK_GRP_DECLARE(fadv_lock_grp, "fadv_lock");
103 static LCK_MTX_DECLARE(fadv_lock, &fadv_lock_grp);
104
105 /* protected by fadv_lock */
106 static STAILQ_HEAD(fadv_head, flowadv_fcentry) fadv_list =
107 STAILQ_HEAD_INITIALIZER(fadv_list);
108 static thread_t fadv_thread = THREAD_NULL;
109 static uint32_t fadv_active;
110
111 static const unsigned int fadv_size = sizeof(struct flowadv_fcentry);
112 static struct mcache *fadv_cache; /* mcache for flowadv_fcentry */
113
114 #define FADV_CACHE_NAME "flowadv" /* cache name */
115
116 static int flowadv_thread_cont(int);
117 static void flowadv_thread_func(void *, wait_result_t);
118
119 void
flowadv_init(void)120 flowadv_init(void)
121 {
122 fadv_cache = mcache_create(FADV_CACHE_NAME, fadv_size,
123 sizeof(uint64_t), 0, MCR_SLEEP);
124
125 if (kernel_thread_start(flowadv_thread_func, NULL, &fadv_thread) !=
126 KERN_SUCCESS) {
127 panic("%s: couldn't create flow event advisory thread",
128 __func__);
129 /* NOTREACHED */
130 }
131 thread_deallocate(fadv_thread);
132 }
133
134 struct flowadv_fcentry *
flowadv_alloc_entry(int how)135 flowadv_alloc_entry(int how)
136 {
137 struct flowadv_fcentry *fce;
138
139 if ((fce = mcache_alloc(fadv_cache, (how == M_WAITOK) ?
140 MCR_SLEEP : MCR_NOSLEEP)) != NULL) {
141 bzero(fce, fadv_size);
142 }
143
144 return fce;
145 }
146
147 void
flowadv_free_entry(struct flowadv_fcentry * fce)148 flowadv_free_entry(struct flowadv_fcentry *fce)
149 {
150 mcache_free(fadv_cache, fce);
151 }
152
153 void
flowadv_add(struct flowadv_fclist * fcl)154 flowadv_add(struct flowadv_fclist *fcl)
155 {
156 if (STAILQ_EMPTY(fcl)) {
157 return;
158 }
159
160 lck_mtx_lock_spin(&fadv_lock);
161
162 STAILQ_CONCAT(&fadv_list, fcl);
163 VERIFY(!STAILQ_EMPTY(&fadv_list));
164
165 if (!fadv_active && fadv_thread != THREAD_NULL) {
166 wakeup_one((caddr_t)&fadv_list);
167 }
168
169 lck_mtx_unlock(&fadv_lock);
170 }
171
172 void
flowadv_add_entry(struct flowadv_fcentry * fce)173 flowadv_add_entry(struct flowadv_fcentry *fce)
174 {
175 lck_mtx_lock_spin(&fadv_lock);
176 STAILQ_INSERT_HEAD(&fadv_list, fce, fce_link);
177 VERIFY(!STAILQ_EMPTY(&fadv_list));
178
179 if (!fadv_active && fadv_thread != THREAD_NULL) {
180 wakeup_one((caddr_t)&fadv_list);
181 }
182
183 lck_mtx_unlock(&fadv_lock);
184 }
185
186 static int
flowadv_thread_cont(int err)187 flowadv_thread_cont(int err)
188 {
189 #pragma unused(err)
190 for (;;) {
191 LCK_MTX_ASSERT(&fadv_lock, LCK_MTX_ASSERT_OWNED);
192 while (STAILQ_EMPTY(&fadv_list)) {
193 VERIFY(!fadv_active);
194 (void) msleep0(&fadv_list, &fadv_lock, (PSOCK | PSPIN),
195 "flowadv_cont", 0, flowadv_thread_cont);
196 /* NOTREACHED */
197 }
198
199 fadv_active = 1;
200 for (;;) {
201 struct flowadv_fcentry *fce;
202
203 VERIFY(!STAILQ_EMPTY(&fadv_list));
204 fce = STAILQ_FIRST(&fadv_list);
205 STAILQ_REMOVE(&fadv_list, fce,
206 flowadv_fcentry, fce_link);
207 STAILQ_NEXT(fce, fce_link) = NULL;
208
209 lck_mtx_unlock(&fadv_lock);
210 switch (fce->fce_flowsrc_type) {
211 case FLOWSRC_INPCB:
212 inp_flowadv(fce->fce_flowid);
213 break;
214
215 case FLOWSRC_IFNET:
216 #if SKYWALK
217 /*
218 * when using the flowID allocator, IPSec
219 * driver uses the "pkt_flowid" field in mbuf
220 * packet header for the globally unique flowID
221 * and the "pkt_mpriv_srcid" field carries the
222 * interface flow control id (if_flowhash).
223 * For IPSec flows, it is the IPSec driver
224 * network interface which is flow controlled,
225 * instead of the IPSec SA flow.
226 */
227 ifnet_flowadv(fce->fce_flowsrc_token);
228 #else /* !SKYWALK */
229 ifnet_flowadv(fce->fce_flowid);
230 #endif /* !SKYWALK */
231 break;
232
233 #if SKYWALK
234 case FLOWSRC_CHANNEL:
235 kern_channel_flowadv_clear(fce);
236 break;
237 #endif /* SKYWALK */
238
239 case FLOWSRC_PF:
240 default:
241 break;
242 }
243 flowadv_free_entry(fce);
244 lck_mtx_lock_spin(&fadv_lock);
245
246 /* if there's no pending request, we're done */
247 if (STAILQ_EMPTY(&fadv_list)) {
248 break;
249 }
250 }
251 fadv_active = 0;
252 }
253 }
254
255 __dead2
256 static void
flowadv_thread_func(void * v,wait_result_t w)257 flowadv_thread_func(void *v, wait_result_t w)
258 {
259 #pragma unused(v, w)
260 lck_mtx_lock(&fadv_lock);
261 (void) msleep0(&fadv_list, &fadv_lock, (PSOCK | PSPIN),
262 "flowadv", 0, flowadv_thread_cont);
263 /*
264 * msleep0() shouldn't have returned as PCATCH was not set;
265 * therefore assert in this case.
266 */
267 lck_mtx_unlock(&fadv_lock);
268 VERIFY(0);
269 }
270
271 void
flowadv_reap_caches(boolean_t purge)272 flowadv_reap_caches(boolean_t purge)
273 {
274 mcache_reap_now(fadv_cache, purge);
275 }
276