Lines Matching refs:fq
134 fq_t *fq = NULL; in fq_alloc() local
135 fq = mcache_alloc(flowq_cache, MCR_SLEEP); in fq_alloc()
136 if (fq == NULL) { in fq_alloc()
141 bzero(fq, flowq_size); in fq_alloc()
143 MBUFQ_INIT(&fq->fq_mbufq); in fq_alloc()
148 KPKTQ_INIT(&fq->fq_kpktq); in fq_alloc()
151 CLASSQ_PKT_INIT(&fq->fq_dq_head); in fq_alloc()
152 CLASSQ_PKT_INIT(&fq->fq_dq_tail); in fq_alloc()
153 fq->fq_in_dqlist = false; in fq_alloc()
154 return fq; in fq_alloc()
158 fq_destroy(fq_t *fq, classq_pkt_type_t ptype) in fq_destroy() argument
160 VERIFY(!fq->fq_in_dqlist); in fq_destroy()
161 VERIFY(fq_empty(fq, ptype)); in fq_destroy()
162 VERIFY(!(fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW | in fq_destroy()
164 VERIFY(fq->fq_bytes == 0); in fq_destroy()
165 mcache_free(flowq_cache, fq); in fq_destroy()
199 fq_head_drop(fq_if_t *fqs, fq_t *fq) in fq_head_drop() argument
207 fq_getq_flow_internal(fqs, fq, &pkt); in fq_head_drop()
239 fq_compressor(fq_if_t *fqs, fq_t *fq, fq_if_classq_t *fq_cl, in fq_compressor() argument
262 if (fq_empty(fq, fqs->fqs_ptype)) { in fq_compressor()
267 struct mbuf *m = MBUFQ_LAST(&fq->fq_mbufq); in fq_compressor()
274 MBUFQ_REMOVE(&fq->fq_mbufq, m); in fq_compressor()
283 struct __kern_packet *kpkt = KPKTQ_LAST(&fq->fq_kpktq); in fq_compressor()
290 KPKTQ_REMOVE(&fq->fq_kpktq, kpkt); in fq_compressor()
301 fq->fq_bytes -= old_pktlen; in fq_compressor()
307 FQ_GRP_DEC_LEN(fq); in fq_compressor()
308 FQ_GRP_DEC_BYTES(fq, old_pktlen); in fq_compressor()
321 fq_t *fq = NULL; in fq_addq() local
362 fq = fq_if_hash_pkt(fqs, fq_grp, pkt_flowid, pktsched_get_pkt_svc(pkt), in fq_addq()
364 if (__improbable(fq == NULL)) { in fq_addq()
370 VERIFY(fq->fq_group == fq_grp); in fq_addq()
373 KDBG(AQM_KTRACE_STATS_FLOW_ENQUEUE, fq->fq_flowhash, in fq_addq()
374 AQM_KTRACE_FQ_GRP_SC_IDX(fq), in fq_addq()
375 fq->fq_bytes, pktsched_get_pkt_len(pkt)); in fq_addq()
377 fq_detect_dequeue_stall(fqs, fq, fq_cl, &now); in fq_addq()
379 if (__improbable(FQ_IS_DELAY_HIGH(fq) || FQ_IS_OVERWHELMING(fq))) { in fq_addq()
380 if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) && in fq_addq()
394 fq_if_classq_t *, fq_cl, fq_t *, fq, in fq_addq()
403 if (!fq_empty(fq, fqs->fqs_ptype)) { in fq_addq()
407 fq_head_drop(fqs, fq); in fq_addq()
416 fq_if_classq_t *, fq_cl, fq_t *, fq, in fq_addq()
424 if (fq_if_add_fcentry(fqs, pkt, pkt_flowsrc, fq, fq_cl)) { in fq_addq()
425 fq->fq_flags |= FQF_FLOWCTL_ON; in fq_addq()
455 if (fqs->fqs_large_flow == fq) { in fq_addq()
462 fq_if_classq_t *, fq_cl, fq_t *, fq, in fq_addq()
466 fq_head_drop(fqs, fq); in fq_addq()
474 if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) && in fq_addq()
478 if (fq_if_add_fcentry(fqs, pkt, pkt_flowsrc, fq, fq_cl)) { in fq_addq()
479 fq->fq_flags |= FQF_FLOWCTL_ON; in fq_addq()
480 FQ_SET_OVERWHELMING(fq); in fq_addq()
493 fq_if_classq_t *, fq_cl, fq_t *, fq, in fq_addq()
500 if (fq_empty(fq, fqs->fqs_ptype) && in fq_addq()
501 !(fq->fq_flags & (FQF_NEW_FLOW | in fq_addq()
504 fq, now); in fq_addq()
505 fq = NULL; in fq_addq()
510 fq_t *, fq, pktsched_pkt_t *, pkt, in fq_addq()
529 ret = fq_compressor(fqs, fq, fq_cl, pkt); in fq_addq()
537 fq_t *, fq, pktsched_pkt_t *, pkt, uint32_t, cnt); in fq_addq()
538 fq_enqueue(fq, pkt->pktsched_pkt, pkt->pktsched_tail, cnt, in fq_addq()
541 fq->fq_bytes += chain_len; in fq_addq()
549 fq_if_is_flow_heavy(fqs, fq); in fq_addq()
559 if ((fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW)) == 0) { in fq_addq()
560 VERIFY(STAILQ_NEXT(fq, fq_actlink) == NULL); in fq_addq()
561 STAILQ_INSERT_TAIL(&fq_cl->fcl_new_flows, fq, fq_actlink); in fq_addq()
562 fq->fq_flags |= FQF_NEW_FLOW; in fq_addq()
566 fq->fq_deficit = fq_cl->fcl_quantum; in fq_addq()
572 fq_getq_flow_internal(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt) in fq_getq_flow_internal() argument
579 fq_dequeue(fq, &p, fqs->fqs_ptype); in fq_getq_flow_internal()
588 VERIFY(fq->fq_bytes >= plen); in fq_getq_flow_internal()
589 fq->fq_bytes -= plen; in fq_getq_flow_internal()
591 fq_cl = &FQ_CLASSQ(fq); in fq_getq_flow_internal()
597 FQ_GRP_DEC_LEN(fq); in fq_getq_flow_internal()
598 FQ_GRP_DEC_BYTES(fq, plen); in fq_getq_flow_internal()
601 if (fq_empty(fq, fqs->fqs_ptype)) { in fq_getq_flow_internal()
602 fq->fq_getqtime = 0; in fq_getq_flow_internal()
607 fq_getq_flow(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt, uint64_t now) in fq_getq_flow() argument
614 fq_getq_flow_internal(fqs, fq, pkt); in fq_getq_flow()
627 fq_cl = &FQ_CLASSQ(fq); in fq_getq_flow()
629 if (fq->fq_min_qdelay == 0 || in fq_getq_flow()
630 (qdelay > 0 && (u_int64_t)qdelay < fq->fq_min_qdelay)) { in fq_getq_flow()
631 fq->fq_min_qdelay = qdelay; in fq_getq_flow()
657 "flow: 0x%x, iface: %s", __func__, fq->fq_flowhash, in fq_getq_flow()
671 if (now >= fq->fq_updatetime) { in fq_getq_flow()
672 if (fq->fq_min_qdelay > FQ_TARGET_DELAY(fq)) { in fq_getq_flow()
673 if (!FQ_IS_DELAY_HIGH(fq)) { in fq_getq_flow()
674 FQ_SET_DELAY_HIGH(fq); in fq_getq_flow()
677 "iface: %s", __func__, fq->fq_sc_index, in fq_getq_flow()
678 fq->fq_min_qdelay, fq->fq_flowhash, in fq_getq_flow()
682 FQ_CLEAR_DELAY_HIGH(fq); in fq_getq_flow()
685 fq->fq_updatetime = now + FQ_UPDATE_INTERVAL(fq); in fq_getq_flow()
686 fq->fq_min_qdelay = 0; in fq_getq_flow()
688 if (fqs->fqs_large_flow != fq || !fq_if_almost_at_drop_limit(fqs)) { in fq_getq_flow()
689 FQ_CLEAR_OVERWHELMING(fq); in fq_getq_flow()
691 if (!FQ_IS_DELAY_HIGH(fq) || fq_empty(fq, fqs->fqs_ptype)) { in fq_getq_flow()
692 FQ_CLEAR_DELAY_HIGH(fq); in fq_getq_flow()
695 if ((fq->fq_flags & FQF_FLOWCTL_ON) && in fq_getq_flow()
696 !FQ_IS_DELAY_HIGH(fq) && !FQ_IS_OVERWHELMING(fq)) { in fq_getq_flow()
697 fq_if_flow_feedback(fqs, fq, fq_cl); in fq_getq_flow()
700 if (fq_empty(fq, fqs->fqs_ptype)) { in fq_getq_flow()
702 fq->fq_getqtime = 0; in fq_getq_flow()
704 fq->fq_getqtime = now; in fq_getq_flow()
706 fq_if_is_flow_heavy(fqs, fq); in fq_getq_flow()