88 	fq_t *fq = NULL;  in fq_alloc()  local
89 	fq = mcache_alloc(flowq_cache, MCR_SLEEP);  in fq_alloc()
90 	if (fq == NULL) {  in fq_alloc()
95 	bzero(fq, flowq_size);  in fq_alloc()
96 	fq->fq_ptype = ptype;  in fq_alloc()
98 		MBUFQ_INIT(&fq->fq_mbufq);  in fq_alloc()
103 		KPKTQ_INIT(&fq->fq_kpktq);  in fq_alloc()
106 	CLASSQ_PKT_INIT(&fq->fq_dq_head);  in fq_alloc()
107 	CLASSQ_PKT_INIT(&fq->fq_dq_tail);  in fq_alloc()
108 	fq->fq_in_dqlist = false;  in fq_alloc()
109 	return fq;  in fq_alloc()
113 fq_destroy(fq_t *fq)  in fq_destroy()  argument
115 	VERIFY(fq->fq_flags & FQF_DESTROYED);  in fq_destroy()
116 	VERIFY(fq_empty(fq));  in fq_destroy()
117 	VERIFY(!(fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW)));  in fq_destroy()
118 	VERIFY(fq->fq_bytes == 0);  in fq_destroy()
119 	mcache_free(flowq_cache, fq);  in fq_destroy()
148 fq_head_drop(fq_if_t *fqs, fq_t *fq)  in fq_head_drop()  argument
156 	fq_getq_flow_internal(fqs, fq, &pkt);  in fq_head_drop()
188 fq_compressor(fq_if_t *fqs, fq_t *fq, fq_if_classq_t *fq_cl,  in fq_compressor()  argument
191 	classq_pkt_type_t ptype = fq->fq_ptype;  in fq_compressor()
211 	if (fq_empty(fq)) {  in fq_compressor()
216 		struct mbuf *m = MBUFQ_LAST(&fq->fq_mbufq);  in fq_compressor()
223 		MBUFQ_REMOVE(&fq->fq_mbufq, m);  in fq_compressor()
232 		struct __kern_packet *kpkt = KPKTQ_LAST(&fq->fq_kpktq);  in fq_compressor()
239 		KPKTQ_REMOVE(&fq->fq_kpktq, kpkt);  in fq_compressor()
250 	fq->fq_bytes -= old_pktlen;  in fq_compressor()
266 	fq_t *fq = NULL;  in fq_addq()  local
306 	fq = fq_if_hash_pkt(fqs, pkt_flowid, pktsched_get_pkt_svc(pkt),  in fq_addq()
308 	if (__improbable(fq == NULL)) {  in fq_addq()
314 	VERIFY(fq->fq_ptype == pkt->pktsched_ptype);  in fq_addq()
316 	fq_detect_dequeue_stall(fqs, fq, fq_cl, &now);  in fq_addq()
318 	if (__improbable(FQ_IS_DELAYHIGH(fq) || FQ_IS_OVERWHELMING(fq))) {  in fq_addq()
319 		if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) &&  in fq_addq()
333 			    fq_if_classq_t *, fq_cl, fq_t *, fq,  in fq_addq()
342 			if (!fq_empty(fq)) {  in fq_addq()
346 					fq_head_drop(fqs, fq);  in fq_addq()
355 			    fq_if_classq_t *, fq_cl, fq_t *, fq,  in fq_addq()
363 		if (fq_if_add_fcentry(fqs, pkt, pkt_flowsrc, fq, fq_cl)) {  in fq_addq()
364 			fq->fq_flags |= FQF_FLOWCTL_ON;  in fq_addq()
394 		if (fqs->fqs_large_flow == fq) {  in fq_addq()
401 			    fq_if_classq_t *, fq_cl, fq_t *, fq,  in fq_addq()
405 				fq_head_drop(fqs, fq);  in fq_addq()
413 			if ((fq->fq_flags & FQF_FLOWCTL_CAPABLE) &&  in fq_addq()
417 				if (fq_if_add_fcentry(fqs, pkt, pkt_flowsrc, fq, fq_cl)) {  in fq_addq()
418 					fq->fq_flags |= FQF_FLOWCTL_ON;  in fq_addq()
419 					FQ_SET_OVERWHELMING(fq);  in fq_addq()
432 				    fq_if_classq_t *, fq_cl, fq_t *, fq,  in fq_addq()
439 				if (fq_empty(fq) && !(fq->fq_flags &  in fq_addq()
441 					fq_if_destroy_flow(fqs, fq_cl, fq, true);  in fq_addq()
442 					fq = NULL;  in fq_addq()
447 				    fq_t *, fq, pktsched_pkt_t *, pkt,  in fq_addq()
466 			ret = fq_compressor(fqs, fq, fq_cl, pkt);  in fq_addq()
474 		    fq_t *, fq, pktsched_pkt_t *, pkt, uint32_t, cnt);  in fq_addq()
475 		fq_enqueue(fq, pkt->pktsched_pkt, pkt->pktsched_tail, cnt);  in fq_addq()
477 		fq->fq_bytes += chain_len;  in fq_addq()
485 		fq_if_is_flow_heavy(fqs, fq);  in fq_addq()
495 	if ((fq->fq_flags & (FQF_NEW_FLOW | FQF_OLD_FLOW)) == 0) {  in fq_addq()
496 		VERIFY(STAILQ_NEXT(fq, fq_actlink) == NULL);  in fq_addq()
497 		STAILQ_INSERT_TAIL(&fq_cl->fcl_new_flows, fq, fq_actlink);  in fq_addq()
498 		fq->fq_flags |= FQF_NEW_FLOW;  in fq_addq()
502 		fq->fq_deficit = fq_cl->fcl_quantum;  in fq_addq()
508 fq_getq_flow_internal(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt)  in fq_getq_flow_internal()  argument
515 	fq_dequeue(fq, &p);  in fq_getq_flow_internal()
524 	VERIFY(fq->fq_bytes >= plen);  in fq_getq_flow_internal()
525 	fq->fq_bytes -= plen;  in fq_getq_flow_internal()
527 	fq_cl = &fqs->fqs_classq[fq->fq_sc_index];  in fq_getq_flow_internal()
534 	if (fq_empty(fq)) {  in fq_getq_flow_internal()
535 		fq->fq_getqtime = 0;  in fq_getq_flow_internal()
540 fq_getq_flow(fq_if_t *fqs, fq_t *fq, pktsched_pkt_t *pkt)  in fq_getq_flow()  argument
549 	fq_getq_flow_internal(fqs, fq, pkt);  in fq_getq_flow()
565 	fq_cl = &fqs->fqs_classq[fq->fq_sc_index];  in fq_getq_flow()
567 	if (fq->fq_min_qdelay == 0 ||  in fq_getq_flow()
568 	    (qdelay > 0 && (u_int64_t)qdelay < fq->fq_min_qdelay)) {  in fq_getq_flow()
569 		fq->fq_min_qdelay = qdelay;  in fq_getq_flow()
595 			    "flow: 0x%x, iface: %s", __func__, fq->fq_flowhash,  in fq_getq_flow()
609 	if (now >= fq->fq_updatetime) {  in fq_getq_flow()
610 		if (fq->fq_min_qdelay > fqs->fqs_target_qdelay) {  in fq_getq_flow()
611 			if (!FQ_IS_DELAYHIGH(fq)) {  in fq_getq_flow()
612 				FQ_SET_DELAY_HIGH(fq);  in fq_getq_flow()
615 				    "iface: %s", __func__, fq->fq_sc_index,  in fq_getq_flow()
616 				    fq->fq_min_qdelay, fq->fq_flowhash,  in fq_getq_flow()
620 			FQ_CLEAR_DELAY_HIGH(fq);  in fq_getq_flow()
623 		fq->fq_updatetime = now + fqs->fqs_update_interval;  in fq_getq_flow()
624 		fq->fq_min_qdelay = 0;  in fq_getq_flow()
627 	if (fqs->fqs_large_flow != fq || !fq_if_almost_at_drop_limit(fqs)) {  in fq_getq_flow()
628 		FQ_CLEAR_OVERWHELMING(fq);  in fq_getq_flow()
630 	if (!FQ_IS_DELAYHIGH(fq) || fq_empty(fq)) {  in fq_getq_flow()
631 		FQ_CLEAR_DELAY_HIGH(fq);  in fq_getq_flow()
634 	if ((fq->fq_flags & FQF_FLOWCTL_ON) &&  in fq_getq_flow()
635 	    !FQ_IS_DELAYHIGH(fq) && !FQ_IS_OVERWHELMING(fq)) {  in fq_getq_flow()
636 		fq_if_flow_feedback(fqs, fq, fq_cl);  in fq_getq_flow()
639 	if (fq_empty(fq)) {  in fq_getq_flow()
641 		fq->fq_getqtime = 0;  in fq_getq_flow()
643 		fq->fq_getqtime = now;  in fq_getq_flow()
645 	fq_if_is_flow_heavy(fqs, fq);  in fq_getq_flow()