1 /* 2 * Copyright (c) 2016-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _NET_CLASSQ_CLASSQ_FQ_CODEL_H 30 #define _NET_CLASSQ_CLASSQ_FQ_CODEL_H 31 #ifdef PRIVATE 32 #ifdef BSD_KERNEL_PRIVATE 33 #include <stdbool.h> 34 #include <sys/time.h> 35 #include <net/flowadv.h> 36 #include <net/classq/if_classq.h> 37 #if SKYWALK 38 #include <skywalk/os_skywalk_private.h> 39 #endif /* SKYWALK */ 40 41 #ifdef __cplusplus 42 extern "C" { 43 #endif 44 45 #define FQ_MIN_FC_THRESHOLD_BYTES 7500 46 #define FQ_IS_DELAYHIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) 47 #define FQ_SET_DELAY_HIGH(_fq_) do { \ 48 (_fq_)->fq_flags |= FQF_DELAY_HIGH; \ 49 } while (0) 50 #define FQ_CLEAR_DELAY_HIGH(_fq_) do { \ 51 (_fq_)->fq_flags &= ~FQF_DELAY_HIGH; \ 52 } while (0) 53 54 #define FQ_IS_OVERWHELMING(_fq_) ((_fq_)->fq_flags & FQF_OVERWHELMING) 55 #define FQ_SET_OVERWHELMING(_fq_) do { \ 56 (_fq_)->fq_flags |= FQF_OVERWHELMING; \ 57 } while (0) 58 #define FQ_CLEAR_OVERWHELMING(_fq_) do { \ 59 (_fq_)->fq_flags &= ~FQF_OVERWHELMING; \ 60 } while (0) 61 62 typedef struct flowq { 63 union { 64 MBUFQ_HEAD(mbufq_head) __mbufq; /* mbuf packet queue */ 65 #if SKYWALK 66 KPKTQ_HEAD(kpktq_head) __kpktq; /* skywalk packet queue */ 67 #endif /* SKYWALK */ 68 } __fq_pktq_u; 69 #define FQF_FLOWCTL_CAPABLE 0x01 /* Use flow control instead of drop */ 70 #define FQF_DELAY_HIGH 0x02 /* Min delay is greater than target */ 71 #define FQF_NEW_FLOW 0x04 /* Currently on new flows queue */ 72 #define FQF_OLD_FLOW 0x08 /* Currently on old flows queue */ 73 #define FQF_FLOWCTL_ON 0x10 /* Currently flow controlled */ 74 #define FQF_DESTROYED 0x80 /* flowq destroyed */ 75 #define FQF_OVERWHELMING 0x40 /* The largest flow when AQM hits queue limit */ 76 uint8_t fq_flags; /* flags */ 77 uint8_t fq_sc_index; /* service_class index */ 78 int32_t fq_deficit; /* Deficit for scheduling */ 79 uint32_t fq_bytes; /* Number of bytes in the queue */ 80 uint64_t fq_min_qdelay; /* min queue delay for Codel */ 81 uint64_t fq_updatetime; /* next update interval */ 82 uint64_t fq_getqtime; /* last dequeue time */ 83 SLIST_ENTRY(flowq) fq_hashlink; /* for flow queue hash table */ 84 STAILQ_ENTRY(flowq) fq_actlink; /* for new/old flow queues */ 85 uint32_t fq_flowhash; /* Flow hash */ 86 classq_pkt_type_t fq_ptype; /* Packet type */ 87 /* temporary packet queue for dequeued packets */ 88 classq_pkt_t fq_dq_head; 89 classq_pkt_t fq_dq_tail; 90 STAILQ_ENTRY(flowq) fq_dqlink; /* entry on dequeue flow list */ 91 bool fq_in_dqlist; 92 } fq_t; 93 94 #define fq_mbufq __fq_pktq_u.__mbufq 95 #if SKYWALK 96 #define fq_kpktq __fq_pktq_u.__kpktq 97 #endif /* SKYWALK */ 98 99 #if SKYWALK 100 #define fq_empty(_q) (((_q)->fq_ptype == QP_MBUF) ? \ 101 MBUFQ_EMPTY(&(_q)->fq_mbufq) : \ 102 KPKTQ_EMPTY(&(_q)->fq_kpktq)) 103 #else /* !SKYWALK */ 104 #define fq_empty(_q) MBUFQ_EMPTY(&(_q)->fq_mbufq) 105 #endif /* !SKYWALK */ 106 107 #if SKYWALK 108 #define fq_enqueue(_q, _h, _t, _c) do { \ 109 switch ((_q)->fq_ptype) { \ 110 case QP_MBUF: \ 111 ASSERT((_h).cp_ptype == QP_MBUF); \ 112 ASSERT((_t).cp_ptype == QP_MBUF); \ 113 MBUFQ_ENQUEUE_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 114 (_t).cp_mbuf); \ 115 MBUFQ_ADD_CRUMB_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 116 (_t).cp_mbuf, PKT_CRUMB_FQ_ENQUEUE); \ 117 break; \ 118 case QP_PACKET: \ 119 ASSERT((_h).cp_ptype == QP_PACKET); \ 120 ASSERT((_t).cp_ptype == QP_PACKET); \ 121 KPKTQ_ENQUEUE_MULTI(&(_q)->fq_kpktq, (_h).cp_kpkt, \ 122 (_t).cp_kpkt, (_c)); \ 123 break; \ 124 default: \ 125 VERIFY(0); \ 126 __builtin_unreachable(); \ 127 break; \ 128 } \ 129 } while (0) 130 #else /* !SKYWALK */ 131 #define fq_enqueue(_q, _h, _t, _c) do { \ 132 MBUFQ_ENQUEUE_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 133 (_t).cp_mbuf); \ 134 MBUFQ_ADD_CRUMB_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 135 (_t).cp_mbuf, PKT_CRUMB_FQ_ENQUEUE); \ 136 } while (0) 137 #endif /* !SKYWALK */ 138 139 #if SKYWALK 140 #define fq_dequeue(_q, _p) do { \ 141 switch ((_q)->fq_ptype) { \ 142 case QP_MBUF: { \ 143 MBUFQ_DEQUEUE(&(_q)->fq_mbufq, (_p)->cp_mbuf); \ 144 if (__probable((_p)->cp_mbuf != NULL)) { \ 145 CLASSQ_PKT_INIT_MBUF((_p), (_p)->cp_mbuf); \ 146 m_add_crumb((_p)->cp_mbuf, \ 147 PKT_CRUMB_FQ_DEQUEUE); \ 148 } \ 149 break; \ 150 } \ 151 case QP_PACKET: { \ 152 KPKTQ_DEQUEUE(&(_q)->fq_kpktq, (_p)->cp_kpkt); \ 153 if (__probable((_p)->cp_kpkt != NULL)) { \ 154 CLASSQ_PKT_INIT_PACKET((_p), (_p)->cp_kpkt); \ 155 } \ 156 break; \ 157 } \ 158 default: \ 159 VERIFY(0); \ 160 __builtin_unreachable(); \ 161 break; \ 162 } \ 163 } while (0) 164 #else /* !SKYWALK */ 165 #define fq_dequeue(_q, _p) do { \ 166 MBUFQ_DEQUEUE(&(_q)->fq_mbufq, (_p)->cp_mbuf); \ 167 if (__probable((_p)->cp_mbuf != NULL)) { \ 168 CLASSQ_PKT_INIT_MBUF((_p), (_p)->cp_mbuf); \ 169 m_add_crumb((_p)->cp_mbuf, PKT_CRUMB_FQ_DEQUEUE); \ 170 } \ 171 } while (0) 172 #endif /* !SKYWALK */ 173 174 struct fq_codel_sched_data; 175 struct fq_if_classq; 176 177 /* Function definitions */ 178 extern void fq_codel_init(void); 179 extern void fq_codel_reap_caches(boolean_t); 180 extern fq_t *fq_alloc(classq_pkt_type_t); 181 extern void fq_destroy(fq_t *); 182 extern int fq_addq(struct fq_codel_sched_data *, pktsched_pkt_t *, 183 struct fq_if_classq *); 184 extern void fq_getq_flow(struct fq_codel_sched_data *, fq_t *, 185 pktsched_pkt_t *); 186 extern void fq_getq_flow_internal(struct fq_codel_sched_data *, 187 fq_t *, pktsched_pkt_t *); 188 extern void fq_head_drop(struct fq_codel_sched_data *, fq_t *); 189 190 #ifdef __cplusplus 191 } 192 #endif 193 #endif /* BSD_KERNEL_PRIVATE */ 194 #endif /* PRIVATE */ 195 #endif /* _NET_CLASSQ_CLASSQ_FQ_CODEL_H */ 196