1 /* 2 * Copyright (c) 2016-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _NET_CLASSQ_CLASSQ_FQ_CODEL_H 30 #define _NET_CLASSQ_CLASSQ_FQ_CODEL_H 31 #ifdef PRIVATE 32 #ifdef BSD_KERNEL_PRIVATE 33 #include <stdbool.h> 34 #include <sys/time.h> 35 #include <net/flowadv.h> 36 #include <net/classq/if_classq.h> 37 #if SKYWALK 38 #include <skywalk/os_skywalk_private.h> 39 #endif /* SKYWALK */ 40 41 #ifdef __cplusplus 42 extern "C" { 43 #endif 44 45 #define AQM_KTRACE_AON_FLOW_HIGH_DELAY AQMDBG_CODE(DBG_AQM_ALWAYSON, 0x001) 46 #define AQM_KTRACE_AON_THROTTLE AQMDBG_CODE(DBG_AQM_ALWAYSON, 0x002) 47 #define AQM_KTRACE_AON_FLOW_OVERWHELMING AQMDBG_CODE(DBG_AQM_ALWAYSON, 0x003) 48 #define AQM_KTRACE_AON_FLOW_DQ_STALL AQMDBG_CODE(DBG_AQM_ALWAYSON, 0x004) 49 50 #define AQM_KTRACE_STATS_FLOW_ENQUEUE AQMDBG_CODE(DBG_AQM_STATS, 0x001) 51 #define AQM_KTRACE_STATS_FLOW_DEQUEUE AQMDBG_CODE(DBG_AQM_STATS, 0x002) 52 #define AQM_KTRACE_STATS_FLOW_CTL AQMDBG_CODE(DBG_AQM_STATS, 0x003) 53 #define AQM_KTRACE_STATS_FLOW_ALLOC AQMDBG_CODE(DBG_AQM_STATS, 0x004) 54 #define AQM_KTRACE_STATS_FLOW_DESTROY AQMDBG_CODE(DBG_AQM_STATS, 0x005) 55 #define AQM_KTRACE_STATS_FLOW_REPORT_CE AQMDBG_CODE(DBG_AQM_STATS, 0x006) 56 #define AQM_KTRACE_STATS_GET_QLEN AQMDBG_CODE(DBG_AQM_STATS, 0x007) 57 #define AQM_KTRACE_TX_NOT_READY AQMDBG_CODE(DBG_AQM_STATS, 0x008) 58 #define AQM_KTRACE_TX_PACEMAKER AQMDBG_CODE(DBG_AQM_STATS, 0x009) 59 #define AQM_KTRACE_PKT_DROP AQMDBG_CODE(DBG_AQM_STATS, 0x00a) 60 #define AQM_KTRACE_OK_TO_DROP AQMDBG_CODE(DBG_AQM_STATS, 0x00b) 61 #define AQM_KTRACE_CONGESTION_INC AQMDBG_CODE(DBG_AQM_STATS, 0x00c) 62 #define AQM_KTRACE_CONGESTION_NOTIFIED AQMDBG_CODE(DBG_AQM_STATS, 0x00d) 63 64 #define AQM_KTRACE_FQ_GRP_SC_IDX(_fq_) \ 65 ((_fq_)->fq_group->fqg_index << 4 | (_fq_)->fq_sc_index) 66 67 #define FQ_MIN_FC_THRESHOLD_BYTES 7500 68 #define FQ_IS_DELAY_HIGH(_fq_) ((_fq_)->fq_flags & FQF_DELAY_HIGH) 69 #define FQ_SET_DELAY_HIGH(_fq_) do { \ 70 if (!FQ_IS_DELAY_HIGH(_fq_)) { \ 71 KDBG(AQM_KTRACE_AON_FLOW_HIGH_DELAY | DBG_FUNC_START, \ 72 (_fq_)->fq_flowhash, AQM_KTRACE_FQ_GRP_SC_IDX((_fq_)), \ 73 (_fq_)->fq_bytes, (_fq_)->fq_min_qdelay); \ 74 } \ 75 (_fq_)->fq_flags |= FQF_DELAY_HIGH; \ 76 } while (0) 77 #define FQ_CLEAR_DELAY_HIGH(_fq_) do { \ 78 if (FQ_IS_DELAY_HIGH(_fq_)) { \ 79 KDBG(AQM_KTRACE_AON_FLOW_HIGH_DELAY | DBG_FUNC_END, \ 80 (_fq_)->fq_flowhash, AQM_KTRACE_FQ_GRP_SC_IDX((_fq_)), \ 81 (_fq_)->fq_bytes, (_fq_)->fq_min_qdelay); \ 82 } \ 83 (_fq_)->fq_flags &= ~FQF_DELAY_HIGH; \ 84 } while (0) 85 86 #define FQ_IS_ECN_CAPABLE(_fq_) ((_fq_)->fq_flags & FQF_ECN_CAPABLE) 87 #define FQ_SET_ECN_CAPABLE(_fq_) do { \ 88 (_fq_)->fq_flags |= FQF_ECN_CAPABLE; \ 89 } while (0) 90 #define FQ_CLEAR_ECN_CAPABLE(_fq_) do { \ 91 (_fq_)->fq_flags &= ~FQF_ECN_CAPABLE; \ 92 } while (0) 93 94 #define FQ_CONGESTION_FEEDBACK_CAPABLE(_fq_) ((_fq_)->fq_flags & FQF_CONGESTION_FEEDBACK) 95 #define FQ_ENABLE_CONGESTION_FEEDBACK(_fq_) do { \ 96 (_fq_)->fq_flags |= FQF_CONGESTION_FEEDBACK; \ 97 } while (0) 98 99 #define FQ_IS_OVERWHELMING(_fq_) ((_fq_)->fq_flags & FQF_OVERWHELMING) 100 #define FQ_SET_OVERWHELMING(_fq_) do { \ 101 if (!FQ_IS_OVERWHELMING(_fq_)) { \ 102 KDBG(AQM_KTRACE_AON_FLOW_OVERWHELMING | DBG_FUNC_START, \ 103 (_fq_)->fq_flowhash, AQM_KTRACE_FQ_GRP_SC_IDX((_fq_)), \ 104 (_fq_)->fq_bytes, (_fq_)->fq_min_qdelay); \ 105 } \ 106 (_fq_)->fq_flags |= FQF_OVERWHELMING; \ 107 } while (0) 108 #define FQ_CLEAR_OVERWHELMING(_fq_) do { \ 109 if (FQ_IS_OVERWHELMING(_fq_)) { \ 110 KDBG(AQM_KTRACE_AON_FLOW_OVERWHELMING | DBG_FUNC_END, \ 111 (_fq_)->fq_flowhash, AQM_KTRACE_FQ_GRP_SC_IDX((_fq_)), \ 112 (_fq_)->fq_bytes, (_fq_)->fq_min_qdelay); \ 113 } \ 114 (_fq_)->fq_flags &= ~FQF_OVERWHELMING; \ 115 } while (0) 116 117 /* 118 * time (in ns) the flow queue can stay in empty state. 119 */ 120 #define FQ_EMPTY_PURGE_DELAY (3ULL * 1000 * 1000 * 1000) 121 122 /* 123 * maximum number of flow queues which can be purged during a dequeue. 124 */ 125 #define FQ_EMPTY_PURGE_MAX 4 126 127 #define FQ_INVALID_TX_TS UINT64_MAX 128 129 typedef struct codel_status { 130 boolean_t dropping; 131 uint32_t lastcnt; 132 uint32_t count; 133 uint64_t first_above_time; 134 uint64_t drop_next; 135 } codel_status_t; 136 137 struct flowq { 138 #pragma pack(push,1) 139 union { 140 MBUFQ_HEAD(mbufq_head) __mbufq; /* mbuf packet queue */ 141 #if SKYWALK 142 KPKTQ_HEAD(kpktq_head) __kpktq; /* skywalk packet queue */ 143 #endif /* SKYWALK */ 144 } __fq_pktq_u; 145 #pragma pack(pop) 146 uint32_t fq_flowhash; /* Flow hash */ 147 uint32_t fq_bytes; /* Number of bytes in the queue */ 148 int32_t fq_deficit; /* Deficit for scheduling */ 149 fq_if_group_t *fq_group; /* Back pointer to the group */ 150 #define FQF_FLOWCTL_CAPABLE 0x01 /* Use flow control instead of drop */ 151 #define FQF_DELAY_HIGH 0x02 /* Min delay is greater than target */ 152 #define FQF_NEW_FLOW 0x04 /* Currently on new flows queue */ 153 #define FQF_OLD_FLOW 0x08 /* Currently on old flows queue */ 154 #define FQF_FLOWCTL_ON 0x10 /* Currently flow controlled */ 155 #define FQF_EMPTY_FLOW 0x20 /* Currently on empty flows queue */ 156 #define FQF_OVERWHELMING 0x40 /* The largest flow when AQM hits queue limit */ 157 #define FQF_FRESH_FLOW 0x80 /* The flow queue has just been allocated */ 158 #define FQF_ECN_CAPABLE 0x100 /* The flow is capable for doing ECN for classic traffic */ 159 #define FQF_CONGESTION_FEEDBACK 0x200 /* The flow is capable for doing congestion feedbacks */ 160 uint16_t fq_flags; /* flags */ 161 uint8_t fq_flowsrc; 162 uint8_t fq_sc_index; /* service_class index */ 163 bool fq_in_dqlist; 164 fq_tfc_type_t fq_tfc_type; 165 uint8_t __fq_pad_uint8[4]; 166 uint64_t fq_min_qdelay; /* min queue delay for Codel */ 167 uint64_t fq_getqtime; /* last dequeue time */ 168 /* total pkt count since last congestion event report */ 169 uint32_t fq_pkts_since_last_report; 170 /* the next time that a paced packet is ready to go*/ 171 uint64_t fq_next_tx_time; 172 /* number of packets that have experienced congestion event */ 173 uint32_t fq_congestion_cnt; 174 uint32_t fq_last_congestion_cnt; 175 codel_status_t codel_status; 176 union { 177 uint64_t fq_updatetime; /* next update interval */ 178 /* empty list purge time (in nanoseconds) */ 179 uint64_t fq_empty_purge_time; 180 }; 181 LIST_ENTRY(flowq) fq_hashlink; /* for flow queue hash table */ 182 /* 183 * flow queue will only be on either one of the lists. 184 */ 185 union { 186 STAILQ_ENTRY(flowq) fq_actlink; /* for new/old flow queues */ 187 /* entry on empty flow queue list */ 188 TAILQ_ENTRY(flowq) fq_empty_link; 189 }; 190 /* entry on dequeue flow list */ 191 STAILQ_ENTRY(flowq) fq_dqlink; 192 /* temporary packet queue for dequeued packets */ 193 classq_pkt_t fq_dq_head; 194 classq_pkt_t fq_dq_tail; 195 }; 196 197 typedef struct flowq fq_t; 198 199 #define FQF_FLOW_STATE_MASK (FQF_DELAY_HIGH | FQF_NEW_FLOW | \ 200 FQF_OLD_FLOW | FQF_FLOWCTL_ON | FQF_EMPTY_FLOW) 201 202 #define fq_mbufq __fq_pktq_u.__mbufq 203 #if SKYWALK 204 #define fq_kpktq __fq_pktq_u.__kpktq 205 #endif /* SKYWALK */ 206 207 #if SKYWALK 208 #define fq_empty(_q, _ptype) (((_ptype) == QP_MBUF) ? \ 209 MBUFQ_EMPTY(&(_q)->fq_mbufq) : KPKTQ_EMPTY(&(_q)->fq_kpktq)) 210 #else /* !SKYWALK */ 211 #define fq_empty(_q, _ptype) MBUFQ_EMPTY(&(_q)->fq_mbufq) 212 #endif /* !SKYWALK */ 213 214 #if SKYWALK 215 #define fq_enqueue(_q, _h, _t, _c, _ptype) do { \ 216 switch (_ptype) { \ 217 case QP_MBUF: \ 218 ASSERT((_h).cp_ptype == QP_MBUF); \ 219 ASSERT((_t).cp_ptype == QP_MBUF); \ 220 MBUFQ_ENQUEUE_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 221 (_t).cp_mbuf); \ 222 MBUFQ_ADD_CRUMB_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 223 (_t).cp_mbuf, PKT_CRUMB_FQ_ENQUEUE); \ 224 break; \ 225 case QP_PACKET: \ 226 ASSERT((_h).cp_ptype == QP_PACKET); \ 227 ASSERT((_t).cp_ptype == QP_PACKET); \ 228 KPKTQ_ENQUEUE_MULTI(&(_q)->fq_kpktq, (_h).cp_kpkt, \ 229 (_t).cp_kpkt, (_c)); \ 230 break; \ 231 default: \ 232 VERIFY(0); \ 233 __builtin_unreachable(); \ 234 break; \ 235 } \ 236 } while (0) 237 #else /* !SKYWALK */ 238 #define fq_enqueue(_q, _h, _t, _c, _ptype) { \ 239 MBUFQ_ENQUEUE_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, (_t).cp_mbuf); \ 240 MBUFQ_ADD_CRUMB_MULTI(&(_q)->fq_mbufq, (_h).cp_mbuf, \ 241 (_t).cp_mbuf, PKT_CRUMB_FQ_ENQUEUE); \ 242 } while (0) 243 #endif /* !SKYWALK */ 244 245 #if SKYWALK 246 #define fq_dequeue(_q, _p, _ptype) do { \ 247 switch (_ptype) { \ 248 case QP_MBUF: { \ 249 MBUFQ_DEQUEUE(&(_q)->fq_mbufq, (_p)->cp_mbuf); \ 250 if (__probable((_p)->cp_mbuf != NULL)) { \ 251 CLASSQ_PKT_INIT_MBUF((_p), (_p)->cp_mbuf); \ 252 m_add_crumb((_p)->cp_mbuf, \ 253 PKT_CRUMB_FQ_DEQUEUE); \ 254 } \ 255 break; \ 256 } \ 257 case QP_PACKET: { \ 258 KPKTQ_DEQUEUE(&(_q)->fq_kpktq, (_p)->cp_kpkt); \ 259 if (__probable((_p)->cp_kpkt != NULL)) { \ 260 CLASSQ_PKT_INIT_PACKET((_p), (_p)->cp_kpkt); \ 261 } \ 262 break; \ 263 } \ 264 default: \ 265 VERIFY(0); \ 266 __builtin_unreachable(); \ 267 break; \ 268 } \ 269 } while (0) 270 #else /* !SKYWALK */ 271 #define fq_dequeue(_q, _p, _ptype) do { \ 272 MBUFQ_DEQUEUE(&(_q)->fq_mbufq, (_p)->cp_mbuf); \ 273 if (__probable((_p)->cp_mbuf != NULL)) { \ 274 CLASSQ_PKT_INIT_MBUF((_p), (_p)->cp_mbuf); \ 275 m_add_crumb((_p)->cp_mbuf, PKT_CRUMB_FQ_DEQUEUE); \ 276 } \ 277 } while (0) 278 #endif /* !SKYWALK */ 279 280 struct fq_codel_sched_data; 281 struct fq_if_classq; 282 283 /* Function definitions */ 284 extern void fq_codel_init(void); 285 extern fq_t *fq_alloc(classq_pkt_type_t); 286 extern void fq_destroy(fq_t *, classq_pkt_type_t); 287 extern int fq_codel_enq_legacy(void * fqs, fq_if_group_t *, pktsched_pkt_t *, struct fq_if_classq *); 288 extern void fq_codel_dq_legacy(void *fqs, void *fq, pktsched_pkt_t *pkt, uint64_t now); 289 extern void fq_getq_flow_internal(struct fq_codel_sched_data *, 290 fq_t *, pktsched_pkt_t *); 291 extern void fq_head_drop(struct fq_codel_sched_data *, fq_t *); 292 extern boolean_t fq_tx_time_ready(fq_if_t *fqs, fq_t *fq, uint64_t now, 293 uint64_t *ready_time); 294 295 extern void fq_codel_dq(void *fqs_p, void *fq_p, pktsched_pkt_t *pkt, uint64_t now); 296 extern int fq_codel_enq(void *fqs_p, fq_if_group_t *fq_grp, pktsched_pkt_t *pkt, 297 fq_if_classq_t *fq_cl); 298 299 #ifdef __cplusplus 300 } 301 #endif 302 #endif /* BSD_KERNEL_PRIVATE */ 303 #endif /* PRIVATE */ 304 #endif /* _NET_CLASSQ_CLASSQ_FQ_CODEL_H */ 305