1 /*
2 * Copyright (c) 2016-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/cdefs.h>
30 #include <sys/param.h>
31 #include <sys/mbuf.h>
32 #include <sys/socket.h>
33 #include <sys/sockio.h>
34 #include <sys/systm.h>
35 #include <sys/sysctl.h>
36 #include <sys/syslog.h>
37 #include <sys/proc.h>
38 #include <sys/errno.h>
39 #include <sys/kernel.h>
40 #include <sys/kauth.h>
41 #include <sys/sdt.h>
42 #include <kern/zalloc.h>
43 #include <netinet/in.h>
44
45 #include <net/classq/classq.h>
46 #include <net/classq/if_classq.h>
47 #include <net/pktsched/pktsched.h>
48 #include <net/pktsched/pktsched_fq_codel.h>
49 #include <net/classq/classq_fq_codel.h>
50
51 #include <netinet/tcp_var.h>
52
53 static uint32_t flowq_size; /* size of flowq */
54 static struct mcache *flowq_cache = NULL; /* mcache for flowq */
55
56 #define FQ_ZONE_MAX (32 * 1024) /* across all interfaces */
57
58 #define DTYPE_NODROP 0 /* no drop */
59 #define DTYPE_FORCED 1 /* a "forced" drop */
60 #define DTYPE_EARLY 2 /* an "unforced" (early) drop */
61
62 void
fq_codel_init(void)63 fq_codel_init(void)
64 {
65 if (flowq_cache != NULL) {
66 return;
67 }
68
69 flowq_size = sizeof(fq_t);
70 flowq_cache = mcache_create("fq.flowq", flowq_size, sizeof(uint64_t),
71 0, MCR_SLEEP);
72 if (flowq_cache == NULL) {
73 panic("%s: failed to allocate flowq_cache", __func__);
74 /* NOTREACHED */
75 __builtin_unreachable();
76 }
77 }
78
79 void
fq_codel_reap_caches(boolean_t purge)80 fq_codel_reap_caches(boolean_t purge)
81 {
82 mcache_reap_now(flowq_cache, purge);
83 }
84
85 fq_t *
fq_alloc(classq_pkt_type_t ptype)86 fq_alloc(classq_pkt_type_t ptype)
87 {
88 fq_t *fq = NULL;
89 fq = mcache_alloc(flowq_cache, MCR_SLEEP);
90 if (fq == NULL) {
91 log(LOG_ERR, "%s: unable to allocate from flowq_cache\n", __func__);
92 return NULL;
93 }
94
95 bzero(fq, flowq_size);
96 fq->fq_ptype = ptype;
97 if (ptype == QP_MBUF) {
98 MBUFQ_INIT(&fq->fq_mbufq);
99 }
100 #if SKYWALK
101 else {
102 VERIFY(ptype == QP_PACKET);
103 KPKTQ_INIT(&fq->fq_kpktq);
104 }
105 #endif /* SKYWALK */
106 CLASSQ_PKT_INIT(&fq->fq_dq_head);
107 CLASSQ_PKT_INIT(&fq->fq_dq_tail);
108 fq->fq_in_dqlist = false;
109 return fq;
110 }
111
112 void
fq_destroy(fq_t * fq)113 fq_destroy(fq_t *fq)
114 {
115 VERIFY(fq->fq_flags & FQF_DESTROYED);
116 VERIFY(fq_empty(fq));
117 VERIFY(!(fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW)));
118 VERIFY(fq->fq_bytes == 0);
119 mcache_free(flowq_cache, fq);
120 }
121
122 static inline void
fq_detect_dequeue_stall(fq_if_t * fqs,fq_t * flowq,fq_if_classq_t * fq_cl,u_int64_t * now)123 fq_detect_dequeue_stall(fq_if_t *fqs, fq_t *flowq, fq_if_classq_t *fq_cl,
124 u_int64_t *now)
125 {
126 u_int64_t maxgetqtime;
127 if (FQ_IS_DELAYHIGH(flowq) || flowq->fq_getqtime == 0 ||
128 fq_empty(flowq) ||
129 flowq->fq_bytes < FQ_MIN_FC_THRESHOLD_BYTES) {
130 return;
131 }
132 maxgetqtime = flowq->fq_getqtime + fqs->fqs_update_interval;
133 if ((*now) > maxgetqtime) {
134 /*
135 * there was no dequeue in an update interval worth of
136 * time. It means that the queue is stalled.
137 */
138 FQ_SET_DELAY_HIGH(flowq);
139 fq_cl->fcl_stat.fcl_dequeue_stall++;
140 os_log_error(OS_LOG_DEFAULT, "%s: dequeue stall num: %d, "
141 "scidx: %d, flow: 0x%x, iface: %s", __func__,
142 fq_cl->fcl_stat.fcl_dequeue_stall, flowq->fq_sc_index,
143 flowq->fq_flowhash, if_name(fqs->fqs_ifq->ifcq_ifp));
144 }
145 }
146
147 void
fq_head_drop(fq_if_t * fqs,fq_t * fq)148 fq_head_drop(fq_if_t *fqs, fq_t *fq)
149 {
150 pktsched_pkt_t pkt;
151 volatile uint32_t *pkt_flags;
152 uint64_t *pkt_timestamp;
153 struct ifclassq *ifq = fqs->fqs_ifq;
154
155 _PKTSCHED_PKT_INIT(&pkt);
156 fq_getq_flow_internal(fqs, fq, &pkt);
157 if (pkt.pktsched_pkt_mbuf == NULL) {
158 return;
159 }
160
161 pktsched_get_pkt_vars(&pkt, &pkt_flags, &pkt_timestamp, NULL, NULL,
162 NULL, NULL);
163
164 *pkt_timestamp = 0;
165 switch (pkt.pktsched_ptype) {
166 case QP_MBUF:
167 *pkt_flags &= ~PKTF_PRIV_GUARDED;
168 break;
169 #if SKYWALK
170 case QP_PACKET:
171 /* sanity check */
172 ASSERT((*pkt_flags & ~PKT_F_COMMON_MASK) == 0);
173 break;
174 #endif /* SKYWALK */
175 default:
176 VERIFY(0);
177 /* NOTREACHED */
178 __builtin_unreachable();
179 }
180
181 IFCQ_DROP_ADD(ifq, 1, pktsched_get_pkt_len(&pkt));
182 IFCQ_CONVERT_LOCK(ifq);
183 pktsched_free_pkt(&pkt);
184 }
185
186
187 static int
fq_compressor(fq_if_t * fqs,fq_t * fq,fq_if_classq_t * fq_cl,pktsched_pkt_t * pkt)188 fq_compressor(fq_if_t *fqs, fq_t *fq, fq_if_classq_t *fq_cl,
189 pktsched_pkt_t *pkt)
190 {
191 classq_pkt_type_t ptype = fq->fq_ptype;
192 uint32_t comp_gencnt = 0;
193 uint64_t *pkt_timestamp;
194 uint64_t old_timestamp = 0;
195 uint32_t old_pktlen = 0;
196 struct ifclassq *ifq = fqs->fqs_ifq;
197
198 if (__improbable(!tcp_do_ack_compression)) {
199 return 0;
200 }
201
202 pktsched_get_pkt_vars(pkt, NULL, &pkt_timestamp, NULL, NULL, NULL,
203 &comp_gencnt);
204
205 if (comp_gencnt == 0) {
206 return 0;
207 }
208
209 fq_cl->fcl_stat.fcl_pkts_compressible++;
210
211 if (fq_empty(fq)) {
212 return 0;
213 }
214
215 if (ptype == QP_MBUF) {
216 struct mbuf *m = MBUFQ_LAST(&fq->fq_mbufq);
217
218 if (comp_gencnt != m->m_pkthdr.comp_gencnt) {
219 return 0;
220 }
221
222 /* If we got until here, we should merge/replace the segment */
223 MBUFQ_REMOVE(&fq->fq_mbufq, m);
224 old_pktlen = m_pktlen(m);
225 old_timestamp = m->m_pkthdr.pkt_timestamp;
226
227 IFCQ_CONVERT_LOCK(fqs->fqs_ifq);
228 m_freem(m);
229 }
230 #if SKYWALK
231 else {
232 struct __kern_packet *kpkt = KPKTQ_LAST(&fq->fq_kpktq);
233
234 if (comp_gencnt != kpkt->pkt_comp_gencnt) {
235 return 0;
236 }
237
238 /* If we got until here, we should merge/replace the segment */
239 KPKTQ_REMOVE(&fq->fq_kpktq, kpkt);
240 old_pktlen = kpkt->pkt_length;
241 old_timestamp = kpkt->pkt_timestamp;
242
243 IFCQ_CONVERT_LOCK(fqs->fqs_ifq);
244 pp_free_packet(*(struct kern_pbufpool **)(uintptr_t)&
245 (((struct __kern_quantum *)kpkt)->qum_pp),
246 (uint64_t)kpkt);
247 }
248 #endif /* SKYWALK */
249
250 fq->fq_bytes -= old_pktlen;
251 fq_cl->fcl_stat.fcl_byte_cnt -= old_pktlen;
252 fq_cl->fcl_stat.fcl_pkt_cnt--;
253 IFCQ_DEC_LEN(ifq);
254 IFCQ_DEC_BYTES(ifq, old_pktlen);
255
256 *pkt_timestamp = old_timestamp;
257
258 return CLASSQEQ_COMPRESSED;
259 }
260
261 int
fq_addq(fq_if_t * fqs,pktsched_pkt_t * pkt,fq_if_classq_t * fq_cl)262 fq_addq(fq_if_t *fqs, pktsched_pkt_t *pkt, fq_if_classq_t *fq_cl)
263 {
264 int droptype = DTYPE_NODROP, fc_adv = 0, ret = CLASSQEQ_SUCCESS;
265 u_int64_t now;
266 fq_t *fq = NULL;
267 uint64_t *pkt_timestamp;
268 volatile uint32_t *pkt_flags;
269 uint32_t pkt_flowid, cnt;
270 uint8_t pkt_proto, pkt_flowsrc;
271
272 cnt = pkt->pktsched_pcnt;
273 pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, &pkt_flowid,
274 &pkt_flowsrc, &pkt_proto, NULL);
275
276 /*
277 * XXX Not walking the chain to set this flag on every packet.
278 * This flag is only used for debugging. Nothing is affected if it's
279 * not set.
280 */
281 switch (pkt->pktsched_ptype) {
282 case QP_MBUF:
283 /* See comments in <rdar://problem/14040693> */
284 VERIFY(!(*pkt_flags & PKTF_PRIV_GUARDED));
285 *pkt_flags |= PKTF_PRIV_GUARDED;
286 break;
287 #if SKYWALK
288 case QP_PACKET:
289 /* sanity check */
290 ASSERT((*pkt_flags & ~PKT_F_COMMON_MASK) == 0);
291 break;
292 #endif /* SKYWALK */
293 default:
294 VERIFY(0);
295 /* NOTREACHED */
296 __builtin_unreachable();
297 }
298
299 /*
300 * Timestamps for every packet must be set prior to entering this path.
301 */
302 now = *pkt_timestamp;
303 ASSERT(now > 0);
304
305 /* find the flowq for this packet */
306 fq = fq_if_hash_pkt(fqs, pkt_flowid, pktsched_get_pkt_svc(pkt),
307 now, TRUE, pkt->pktsched_ptype);
308 if (__improbable(fq == NULL)) {
309 DTRACE_IP1(memfail__drop, fq_if_t *, fqs);
310 /* drop the packet if we could not allocate a flow queue */
311 fq_cl->fcl_stat.fcl_drop_memfailure += cnt;
312 return CLASSQEQ_DROP;
313 }
314 VERIFY(fq->fq_ptype == pkt->pktsched_ptype);
315
316 fq_detect_dequeue_stall(fqs, fq, fq_cl, &now);
317
318 if (__improbable(FQ_IS_DELAYHIGH(fq))) {
319 if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) &&
320 (*pkt_flags & PKTF_FLOW_ADV)) {
321 fc_adv = 1;
322 /*
323 * If the flow is suspended or it is not
324 * TCP/QUIC, drop the chain.
325 */
326 if ((pkt_proto != IPPROTO_TCP) &&
327 (pkt_proto != IPPROTO_QUIC)) {
328 droptype = DTYPE_EARLY;
329 fq_cl->fcl_stat.fcl_drop_early += cnt;
330 }
331 DTRACE_IP6(flow__adv, fq_if_t *, fqs,
332 fq_if_classq_t *, fq_cl, fq_t *, fq,
333 int, droptype, pktsched_pkt_t *, pkt,
334 uint32_t, cnt);
335 } else {
336 /*
337 * Need to drop packets to make room for the new
338 * ones. Try to drop from the head of the queue
339 * instead of the latest packets.
340 */
341 if (!fq_empty(fq)) {
342 uint32_t i;
343
344 for (i = 0; i < cnt; i++) {
345 fq_head_drop(fqs, fq);
346 }
347 droptype = DTYPE_NODROP;
348 } else {
349 droptype = DTYPE_EARLY;
350 }
351 fq_cl->fcl_stat.fcl_drop_early += cnt;
352
353 DTRACE_IP6(no__flow__adv, fq_if_t *, fqs,
354 fq_if_classq_t *, fq_cl, fq_t *, fq,
355 int, droptype, pktsched_pkt_t *, pkt,
356 uint32_t, cnt);
357 }
358 }
359
360 /* Set the return code correctly */
361 if (__improbable(fc_adv == 1 && droptype != DTYPE_FORCED)) {
362 if (fq_if_add_fcentry(fqs, pkt, pkt_flowsrc, fq, fq_cl)) {
363 fq->fq_flags |= FQF_FLOWCTL_ON;
364 /* deliver flow control advisory error */
365 if (droptype == DTYPE_NODROP) {
366 ret = CLASSQEQ_SUCCESS_FC;
367 } else {
368 /* dropped due to flow control */
369 ret = CLASSQEQ_DROP_FC;
370 }
371 } else {
372 /*
373 * if we could not flow control the flow, it is
374 * better to drop
375 */
376 droptype = DTYPE_FORCED;
377 ret = CLASSQEQ_DROP_FC;
378 fq_cl->fcl_stat.fcl_flow_control_fail++;
379 }
380 DTRACE_IP3(fc__ret, fq_if_t *, fqs, int, droptype, int, ret);
381 }
382
383 /*
384 * If the queue length hits the queue limit, drop a chain with the
385 * same number of packets from the front of the queue for a flow with
386 * maximum number of bytes. This will penalize heavy and unresponsive
387 * flows. It will also avoid a tail drop.
388 */
389 if (__improbable(droptype == DTYPE_NODROP &&
390 fq_if_at_drop_limit(fqs))) {
391 uint32_t i;
392
393 if (fqs->fqs_large_flow == fq) {
394 /*
395 * Drop from the head of the current fq. Since a
396 * new packet will be added to the tail, it is ok
397 * to leave fq in place.
398 */
399 DTRACE_IP5(large__flow, fq_if_t *, fqs,
400 fq_if_classq_t *, fq_cl, fq_t *, fq,
401 pktsched_pkt_t *, pkt, uint32_t, cnt);
402
403 for (i = 0; i < cnt; i++) {
404 fq_head_drop(fqs, fq);
405 }
406 } else {
407 if (fqs->fqs_large_flow == NULL) {
408 droptype = DTYPE_FORCED;
409 fq_cl->fcl_stat.fcl_drop_overflow += cnt;
410 ret = CLASSQEQ_DROP;
411
412 DTRACE_IP5(no__large__flow, fq_if_t *, fqs,
413 fq_if_classq_t *, fq_cl, fq_t *, fq,
414 pktsched_pkt_t *, pkt, uint32_t, cnt);
415
416 /*
417 * if this fq was freshly created and there
418 * is nothing to enqueue, free it
419 */
420 if (fq_empty(fq) && !(fq->fq_flags &
421 (FQF_NEW_FLOW | FQF_OLD_FLOW))) {
422 fq_if_destroy_flow(fqs, fq_cl, fq, true);
423 fq = NULL;
424 }
425 } else {
426 DTRACE_IP5(different__large__flow,
427 fq_if_t *, fqs, fq_if_classq_t *, fq_cl,
428 fq_t *, fq, pktsched_pkt_t *, pkt,
429 uint32_t, cnt);
430
431 for (i = 0; i < cnt; i++) {
432 fq_if_drop_packet(fqs);
433 }
434 }
435 }
436 }
437
438 if (__probable(droptype == DTYPE_NODROP)) {
439 uint32_t chain_len = pktsched_get_pkt_len(pkt);
440
441 /*
442 * We do not compress if we are enqueuing a chain.
443 * Traversing the chain to look for acks would defeat the
444 * purpose of batch enqueueing.
445 */
446 if (cnt == 1) {
447 ret = fq_compressor(fqs, fq, fq_cl, pkt);
448 if (ret != CLASSQEQ_COMPRESSED) {
449 ret = CLASSQEQ_SUCCESS;
450 } else {
451 fq_cl->fcl_stat.fcl_pkts_compressed++;
452 }
453 }
454 DTRACE_IP5(fq_enqueue, fq_if_t *, fqs, fq_if_classq_t *, fq_cl,
455 fq_t *, fq, pktsched_pkt_t *, pkt, uint32_t, cnt);
456 fq_enqueue(fq, pkt->pktsched_pkt, pkt->pktsched_tail, cnt);
457
458 fq->fq_bytes += chain_len;
459 fq_cl->fcl_stat.fcl_byte_cnt += chain_len;
460 fq_cl->fcl_stat.fcl_pkt_cnt += cnt;
461
462 /*
463 * check if this queue will qualify to be the next
464 * victim queue
465 */
466 fq_if_is_flow_heavy(fqs, fq);
467 } else {
468 DTRACE_IP3(fq_drop, fq_if_t *, fqs, int, droptype, int, ret);
469 return (ret != CLASSQEQ_SUCCESS) ? ret : CLASSQEQ_DROP;
470 }
471
472 /*
473 * If the queue is not currently active, add it to the end of new
474 * flows list for that service class.
475 */
476 if ((fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW)) == 0) {
477 VERIFY(STAILQ_NEXT(fq, fq_actlink) == NULL);
478 STAILQ_INSERT_TAIL(&fq_cl->fcl_new_flows, fq, fq_actlink);
479 fq->fq_flags |= FQF_NEW_FLOW;
480
481 fq_cl->fcl_stat.fcl_newflows_cnt++;
482
483 fq->fq_deficit = fq_cl->fcl_quantum;
484 }
485 return ret;
486 }
487
488 void
fq_getq_flow_internal(fq_if_t * fqs,fq_t * fq,pktsched_pkt_t * pkt)489 fq_getq_flow_internal(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt)
490 {
491 classq_pkt_t p = CLASSQ_PKT_INITIALIZER(p);
492 uint32_t plen;
493 fq_if_classq_t *fq_cl;
494 struct ifclassq *ifq = fqs->fqs_ifq;
495
496 fq_dequeue(fq, &p);
497 if (p.cp_ptype == QP_INVALID) {
498 VERIFY(p.cp_mbuf == NULL);
499 return;
500 }
501
502 pktsched_pkt_encap(pkt, &p);
503 plen = pktsched_get_pkt_len(pkt);
504
505 VERIFY(fq->fq_bytes >= plen);
506 fq->fq_bytes -= plen;
507
508 fq_cl = &fqs->fqs_classq[fq->fq_sc_index];
509 fq_cl->fcl_stat.fcl_byte_cnt -= plen;
510 fq_cl->fcl_stat.fcl_pkt_cnt--;
511 IFCQ_DEC_LEN(ifq);
512 IFCQ_DEC_BYTES(ifq, plen);
513
514 /* Reset getqtime so that we don't count idle times */
515 if (fq_empty(fq)) {
516 fq->fq_getqtime = 0;
517 }
518 }
519
520 void
fq_getq_flow(fq_if_t * fqs,fq_t * fq,pktsched_pkt_t * pkt)521 fq_getq_flow(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt)
522 {
523 fq_if_classq_t *fq_cl;
524 u_int64_t now;
525 int64_t qdelay = 0;
526 struct timespec now_ts;
527 volatile uint32_t *pkt_flags;
528 uint64_t *pkt_timestamp;
529
530 fq_getq_flow_internal(fqs, fq, pkt);
531 if (pkt->pktsched_ptype == QP_INVALID) {
532 VERIFY(pkt->pktsched_pkt_mbuf == NULL);
533 return;
534 }
535
536 pktsched_get_pkt_vars(pkt, &pkt_flags, &pkt_timestamp, NULL, NULL,
537 NULL, NULL);
538
539 nanouptime(&now_ts);
540 now = (now_ts.tv_sec * NSEC_PER_SEC) + now_ts.tv_nsec;
541
542 /* this will compute qdelay in nanoseconds */
543 if (now > *pkt_timestamp) {
544 qdelay = now - *pkt_timestamp;
545 }
546 fq_cl = &fqs->fqs_classq[fq->fq_sc_index];
547
548 if (fq->fq_min_qdelay == 0 ||
549 (qdelay > 0 && (u_int64_t)qdelay < fq->fq_min_qdelay)) {
550 fq->fq_min_qdelay = qdelay;
551 }
552
553 /* Update min/max/avg qdelay for the respective class */
554 if (fq_cl->fcl_stat.fcl_min_qdelay == 0 ||
555 (qdelay > 0 && (u_int64_t)qdelay < fq_cl->fcl_stat.fcl_min_qdelay)) {
556 fq_cl->fcl_stat.fcl_min_qdelay = qdelay;
557 }
558
559 if (fq_cl->fcl_stat.fcl_max_qdelay == 0 ||
560 (qdelay > 0 && (u_int64_t)qdelay > fq_cl->fcl_stat.fcl_max_qdelay)) {
561 fq_cl->fcl_stat.fcl_max_qdelay = qdelay;
562 }
563
564 uint64_t num_dequeues = fq_cl->fcl_stat.fcl_dequeue;
565
566 if (num_dequeues == 0) {
567 fq_cl->fcl_stat.fcl_avg_qdelay = qdelay;
568 } else if (qdelay > 0) {
569 uint64_t res = 0;
570 if (os_add_overflow(num_dequeues, 1, &res)) {
571 /* Reset the dequeue num and dequeue bytes */
572 fq_cl->fcl_stat.fcl_dequeue = num_dequeues = 0;
573 fq_cl->fcl_stat.fcl_dequeue_bytes = 0;
574 fq_cl->fcl_stat.fcl_avg_qdelay = qdelay;
575 os_log_info(OS_LOG_DEFAULT, "%s: dequeue num overflow, "
576 "flow: 0x%x, iface: %s", __func__, fq->fq_flowhash,
577 if_name(fqs->fqs_ifq->ifcq_ifp));
578 } else {
579 uint64_t product = 0;
580 if (os_mul_overflow(fq_cl->fcl_stat.fcl_avg_qdelay,
581 num_dequeues, &product) || os_add_overflow(product, qdelay, &res)) {
582 fq_cl->fcl_stat.fcl_avg_qdelay = qdelay;
583 } else {
584 fq_cl->fcl_stat.fcl_avg_qdelay = res /
585 (num_dequeues + 1);
586 }
587 }
588 }
589
590 if (now >= fq->fq_updatetime) {
591 if (fq->fq_min_qdelay > fqs->fqs_target_qdelay) {
592 if (!FQ_IS_DELAYHIGH(fq)) {
593 FQ_SET_DELAY_HIGH(fq);
594 os_log_error(OS_LOG_DEFAULT,
595 "%s: high delay idx: %d, %llu, flow: 0x%x, "
596 "iface: %s", __func__, fq->fq_sc_index,
597 fq->fq_min_qdelay, fq->fq_flowhash,
598 if_name(fqs->fqs_ifq->ifcq_ifp));
599 }
600 } else {
601 FQ_CLEAR_DELAY_HIGH(fq);
602 }
603 /* Reset measured queue delay and update time */
604 fq->fq_updatetime = now + fqs->fqs_update_interval;
605 fq->fq_min_qdelay = 0;
606 }
607 if (!FQ_IS_DELAYHIGH(fq) || fq_empty(fq)) {
608 FQ_CLEAR_DELAY_HIGH(fq);
609 if (fq->fq_flags & FQF_FLOWCTL_ON) {
610 fq_if_flow_feedback(fqs, fq, fq_cl);
611 }
612 }
613
614 if (fq_empty(fq)) {
615 /* Reset getqtime so that we don't count idle times */
616 fq->fq_getqtime = 0;
617 } else {
618 fq->fq_getqtime = now;
619 }
620 fq_if_is_flow_heavy(fqs, fq);
621
622 *pkt_timestamp = 0;
623 switch (pkt->pktsched_ptype) {
624 case QP_MBUF:
625 *pkt_flags &= ~PKTF_PRIV_GUARDED;
626 break;
627 #if SKYWALK
628 case QP_PACKET:
629 /* sanity check */
630 ASSERT((*pkt_flags & ~PKT_F_COMMON_MASK) == 0);
631 break;
632 #endif /* SKYWALK */
633 default:
634 VERIFY(0);
635 /* NOTREACHED */
636 __builtin_unreachable();
637 }
638 }
639