1 /* 2 * Copyright (c) 2011-2020 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _NET_CLASSQ_IF_CLASSQ_H_ 30 #define _NET_CLASSQ_IF_CLASSQ_H_ 31 32 #ifdef PRIVATE 33 #define IFCQ_SC_MAX 10 /* max number of queues */ 34 35 #ifdef BSD_KERNEL_PRIVATE 36 #include <net/classq/classq.h> 37 38 /* maximum number of packets stored across all queues */ 39 #define IFCQ_DEFAULT_PKT_DROP_LIMIT 2048 40 41 /* classq request types */ 42 typedef enum cqrq { 43 CLASSQRQ_PURGE = 1, /* purge all packets */ 44 CLASSQRQ_PURGE_SC = 2, /* purge service class (and flow) */ 45 CLASSQRQ_EVENT = 3, /* interface events */ 46 CLASSQRQ_THROTTLE = 4, /* throttle packets */ 47 CLASSQRQ_STAT_SC = 5, /* get service class queue stats */ 48 } cqrq_t; 49 50 /* classq purge_sc request argument */ 51 typedef struct cqrq_purge_sc { 52 mbuf_svc_class_t sc; /* (in) service class */ 53 u_int32_t flow; /* (in) 0 means all flows */ 54 u_int32_t packets; /* (out) purged packets */ 55 u_int32_t bytes; /* (out) purged bytes */ 56 } cqrq_purge_sc_t; 57 58 /* classq throttle request argument */ 59 typedef struct cqrq_throttle { 60 u_int32_t set; /* set or get */ 61 u_int32_t level; /* (in/out) throttling level */ 62 } cqrq_throttle_t; 63 64 /* classq service class stats request argument */ 65 typedef struct cqrq_stat_sc { 66 mbuf_svc_class_t sc; /* (in) service class */ 67 u_int8_t grp_idx; /* group index */ 68 u_int32_t packets; /* (out) packets enqueued */ 69 u_int32_t bytes; /* (out) bytes enqueued */ 70 } cqrq_stat_sc_t; 71 72 /* 73 * A token-bucket regulator limits the rate that a network driver can 74 * dequeue packets from the output queue. Modern cards are able to buffer 75 * a large amount of packets and dequeue too many packets at a time. This 76 * bursty dequeue behavior makes it impossible to schedule packets by 77 * queueing disciplines. A token-bucket is used to control the burst size 78 * in a device independent manner. 79 */ 80 struct tb_regulator { 81 u_int64_t tbr_rate_raw; /* (unscaled) token bucket rate */ 82 u_int32_t tbr_percent; /* token bucket rate in percentage */ 83 int64_t tbr_rate; /* (scaled) token bucket rate */ 84 int64_t tbr_depth; /* (scaled) token bucket depth */ 85 86 int64_t tbr_token; /* (scaled) current token */ 87 int64_t tbr_filluptime; /* (scaled) time to fill up bucket */ 88 u_int64_t tbr_last; /* last time token was updated */ 89 90 /* needed for poll-and-dequeue */ 91 }; 92 93 /* simple token bucket meter profile */ 94 struct tb_profile { 95 u_int64_t rate; /* rate in bit-per-sec */ 96 u_int32_t percent; /* rate in percentage */ 97 u_int32_t depth; /* depth in bytes */ 98 }; 99 100 struct ifclassq; 101 enum cqdq_op; 102 enum cqrq; 103 104 #if DEBUG || DEVELOPMENT 105 extern uint32_t ifclassq_flow_control_adv; 106 #endif /* DEBUG || DEVELOPMENT */ 107 extern uint32_t ifclassq_enable_l4s; 108 extern unsigned int ifclassq_enable_pacing; 109 typedef int (*ifclassq_enq_func)(struct ifclassq *, classq_pkt_t *, 110 boolean_t *); 111 typedef void (*ifclassq_deq_func)(struct ifclassq *, classq_pkt_t *); 112 typedef void (*ifclassq_deq_sc_func)(struct ifclassq *, mbuf_svc_class_t, 113 classq_pkt_t *); 114 typedef int (*ifclassq_deq_multi_func)(struct ifclassq *, u_int32_t, 115 u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *, u_int32_t *); 116 typedef int (*ifclassq_deq_sc_multi_func)(struct ifclassq *, 117 mbuf_svc_class_t, u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, 118 u_int32_t *, u_int32_t *); 119 typedef int (*ifclassq_req_func)(struct ifclassq *, enum cqrq, void *); 120 121 /* 122 * Structure defining a queue for a network interface. 123 */ 124 struct ifclassq { 125 decl_lck_mtx_data(, ifcq_lock); 126 127 os_refcnt_t ifcq_refcnt; 128 struct ifnet *ifcq_ifp; /* back pointer to interface */ 129 u_int32_t ifcq_len; /* packet count */ 130 u_int32_t ifcq_maxlen; 131 struct pktcntr ifcq_xmitcnt; 132 struct pktcntr ifcq_dropcnt; 133 134 u_int32_t ifcq_type; /* scheduler type */ 135 u_int32_t ifcq_flags; /* flags */ 136 u_int32_t ifcq_sflags; /* scheduler flags */ 137 u_int32_t ifcq_target_qdelay; /* target queue delay */ 138 u_int32_t ifcq_bytes; /* bytes count */ 139 u_int32_t ifcq_pkt_drop_limit; 140 /* number of doorbells introduced by pacemaker thread */ 141 uint64_t ifcq_doorbells; 142 void *ifcq_disc; /* for scheduler-specific use */ 143 /* 144 * ifcq_disc_slots[] represents the leaf classes configured for the 145 * corresponding discpline/scheduler, ordered by their corresponding 146 * service class index. Each slot holds the queue ID used to identify 147 * the class instance, as well as the class instance pointer itself. 148 * The latter is used during enqueue and dequeue in order to avoid the 149 * costs associated with looking up the class pointer based on the 150 * queue ID. The queue ID is used when querying the statistics from 151 * user space. 152 * 153 * Avoiding the use of queue ID during enqueue and dequeue is made 154 * possible by virtue of knowing the particular mbuf service class 155 * associated with the packets. The service class index of the 156 * packet is used as the index to ifcq_disc_slots[]. 157 * 158 * ifcq_disc_slots[] therefore also acts as a lookup table which 159 * provides for the mapping between MBUF_SC values and the actual 160 * scheduler classes. 161 */ 162 struct ifclassq_disc_slot { 163 u_int32_t qid; 164 void *cl; 165 } ifcq_disc_slots[IFCQ_SC_MAX]; /* for discipline use */ 166 167 /* token bucket regulator */ 168 struct tb_regulator ifcq_tbr; /* TBR */ 169 }; 170 171 /* ifcq_flags */ 172 #define IFCQF_READY 0x01 /* ifclassq supports discipline */ 173 #define IFCQF_ENABLED 0x02 /* ifclassq is in use */ 174 #define IFCQF_TBR 0x04 /* Token Bucket Regulator is in use */ 175 #define IFCQF_DESTROYED 0x08 /* ifclassq torndown */ 176 177 #define IFCQ_IS_READY(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_READY) 178 #define IFCQ_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_ENABLED) 179 #define IFCQ_TBR_IS_ENABLED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_TBR) 180 #define IFCQ_IS_DESTROYED(_ifcq) ((_ifcq)->ifcq_flags & IFCQF_DESTROYED) 181 182 /* classq enqueue return value */ 183 /* packet has to be dropped */ 184 #define CLASSQEQ_DROP (-1) 185 /* packet successfully enqueued */ 186 #define CLASSQEQ_SUCCESS 0 187 /* packet enqueued; give flow control feedback */ 188 #define CLASSQEQ_SUCCESS_FC 1 189 /* packet needs to be dropped due to flowcontrol; give flow control feedback */ 190 #define CLASSQEQ_DROP_FC 2 191 /* packet needs to be dropped due to suspension; give flow control feedback */ 192 #define CLASSQEQ_DROP_SP 3 193 /* packet has been compressed with another one */ 194 #define CLASSQEQ_COMPRESSED 4 195 196 /* interface event argument for CLASSQRQ_EVENT */ 197 typedef enum cqev { 198 CLASSQ_EV_INIT = 0, 199 CLASSQ_EV_LINK_BANDWIDTH = 1, /* link bandwidth has changed */ 200 CLASSQ_EV_LINK_LATENCY = 2, /* link latency has changed */ 201 CLASSQ_EV_LINK_MTU = 3, /* link MTU has changed */ 202 CLASSQ_EV_LINK_UP = 4, /* link is now up */ 203 CLASSQ_EV_LINK_DOWN = 5, /* link is now down */ 204 } cqev_t; 205 #endif /* BSD_KERNEL_PRIVATE */ 206 207 #define IF_CLASSQ_DEF 0x0 208 #define IF_CLASSQ_LOW_LATENCY 0x1 209 #define IF_CLASSQ_L4S 0x2 210 #define IF_DEFAULT_GRP 0x4 211 212 #define IF_CLASSQ_ALL_GRPS UINT8_MAX 213 214 #include <net/classq/classq.h> 215 #include <net/pktsched/pktsched_fq_codel.h> 216 217 #ifdef __cplusplus 218 extern "C" { 219 #endif 220 struct if_ifclassq_stats { 221 u_int32_t ifqs_len; 222 u_int32_t ifqs_maxlen; 223 uint64_t ifqs_doorbells; 224 struct pktcntr ifqs_xmitcnt; 225 struct pktcntr ifqs_dropcnt; 226 u_int32_t ifqs_scheduler; 227 struct fq_codel_classstats ifqs_fq_codel_stats; 228 } __attribute__((aligned(8))); 229 230 #ifdef __cplusplus 231 } 232 #endif 233 234 #ifdef BSD_KERNEL_PRIVATE 235 /* 236 * For ifclassq lock 237 */ 238 #define IFCQ_LOCK_ASSERT_HELD(_ifcq) \ 239 LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_OWNED) 240 241 #define IFCQ_LOCK_ASSERT_NOTHELD(_ifcq) \ 242 LCK_MTX_ASSERT(&(_ifcq)->ifcq_lock, LCK_MTX_ASSERT_NOTOWNED) 243 244 #define IFCQ_LOCK(_ifcq) \ 245 lck_mtx_lock(&(_ifcq)->ifcq_lock) 246 247 #define IFCQ_LOCK_SPIN(_ifcq) \ 248 lck_mtx_lock_spin(&(_ifcq)->ifcq_lock) 249 250 #define IFCQ_CONVERT_LOCK(_ifcq) do { \ 251 IFCQ_LOCK_ASSERT_HELD(_ifcq); \ 252 lck_mtx_convert_spin(&(_ifcq)->ifcq_lock); \ 253 } while (0) 254 255 #define IFCQ_UNLOCK(_ifcq) \ 256 lck_mtx_unlock(&(_ifcq)->ifcq_lock) 257 258 /* 259 * For ifclassq operations 260 */ 261 #define IFCQ_TBR_DEQUEUE(_ifcq, _p, _idx) do { \ 262 ifclassq_tbr_dequeue(_ifcq, _p, _idx); \ 263 } while (0) 264 265 #define IFCQ_TBR_DEQUEUE_SC(_ifcq, _sc, _p, _idx) do { \ 266 ifclassq_tbr_dequeue_sc(_ifcq, _sc, _p, _idx); \ 267 } while (0) 268 269 #define IFCQ_LEN(_ifcq) ((_ifcq)->ifcq_len) 270 #define IFCQ_QFULL(_ifcq) (IFCQ_LEN(_ifcq) >= (_ifcq)->ifcq_maxlen) 271 #define IFCQ_IS_EMPTY(_ifcq) (IFCQ_LEN(_ifcq) == 0) 272 #define IFCQ_INC_LEN(_ifcq) (IFCQ_LEN(_ifcq)++) 273 #define IFCQ_DEC_LEN(_ifcq) (IFCQ_LEN(_ifcq)--) 274 #define IFCQ_ADD_LEN(_ifcq, _len) (IFCQ_LEN(_ifcq) += (_len)) 275 #define IFCQ_SUB_LEN(_ifcq, _len) (IFCQ_LEN(_ifcq) -= (_len)) 276 #define IFCQ_MAXLEN(_ifcq) ((_ifcq)->ifcq_maxlen) 277 #define IFCQ_SET_MAXLEN(_ifcq, _len) ((_ifcq)->ifcq_maxlen = (_len)) 278 #define IFCQ_TARGET_QDELAY(_ifcq) ((_ifcq)->ifcq_target_qdelay) 279 #define IFCQ_BYTES(_ifcq) ((_ifcq)->ifcq_bytes) 280 #define IFCQ_INC_BYTES(_ifcq, _len) \ 281 ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes + (_len)) 282 #define IFCQ_DEC_BYTES(_ifcq, _len) \ 283 ((_ifcq)->ifcq_bytes = (_ifcq)->ifcq_bytes - (_len)) 284 285 #define IFCQ_XMIT_ADD(_ifcq, _pkt, _len) do { \ 286 PKTCNTR_ADD(&(_ifcq)->ifcq_xmitcnt, _pkt, _len); \ 287 } while (0) 288 289 #define IFCQ_DROP_ADD(_ifcq, _pkt, _len) do { \ 290 PKTCNTR_ADD(&(_ifcq)->ifcq_dropcnt, _pkt, _len); \ 291 } while (0) 292 293 #define IFCQ_PKT_DROP_LIMIT(_ifcq) ((_ifcq)->ifcq_pkt_drop_limit) 294 295 extern int ifclassq_setup(struct ifclassq *, struct ifnet *, uint32_t); 296 extern void ifclassq_teardown(struct ifclassq *); 297 extern int ifclassq_pktsched_setup(struct ifclassq *); 298 extern void ifclassq_set_maxlen(struct ifclassq *, u_int32_t); 299 extern u_int32_t ifclassq_get_maxlen(struct ifclassq *); 300 extern int ifclassq_get_len(struct ifclassq *, mbuf_svc_class_t, 301 u_int8_t, u_int32_t *, u_int32_t *); 302 extern errno_t ifclassq_enqueue(struct ifclassq *, classq_pkt_t *, 303 classq_pkt_t *, u_int32_t, u_int32_t, boolean_t *); 304 extern errno_t ifclassq_dequeue(struct ifclassq *, u_int32_t, u_int32_t, 305 classq_pkt_t *, classq_pkt_t *, u_int32_t *, u_int32_t *, u_int8_t); 306 extern errno_t ifclassq_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, 307 u_int32_t, u_int32_t, classq_pkt_t *, classq_pkt_t *, u_int32_t *, 308 u_int32_t *, u_int8_t); 309 extern void *ifclassq_poll(struct ifclassq *, classq_pkt_type_t *); 310 extern void *ifclassq_poll_sc(struct ifclassq *, mbuf_svc_class_t, 311 classq_pkt_type_t *); 312 extern void ifclassq_update(struct ifclassq *, cqev_t); 313 extern int ifclassq_attach(struct ifclassq *, u_int32_t, void *); 314 extern void ifclassq_detach(struct ifclassq *); 315 extern int ifclassq_getqstats(struct ifclassq *, u_int8_t, u_int32_t, 316 void *, u_int32_t *); 317 extern const char *__null_terminated ifclassq_ev2str(cqev_t); 318 extern int ifclassq_tbr_set(struct ifclassq *, struct tb_profile *, boolean_t); 319 extern void ifclassq_tbr_dequeue(struct ifclassq *, classq_pkt_t *, u_int8_t); 320 extern void ifclassq_tbr_dequeue_sc(struct ifclassq *, mbuf_svc_class_t, 321 classq_pkt_t *, u_int8_t); 322 extern void ifclassq_calc_target_qdelay(struct ifnet *ifp, 323 uint64_t *if_target_qdelay, uint32_t flags); 324 extern void ifclassq_calc_update_interval(uint64_t *update_interval, 325 uint32_t flags); 326 extern void ifclassq_set_packet_metadata(struct ifclassq *ifq, 327 struct ifnet *ifp, classq_pkt_t *p); 328 extern struct ifclassq *ifclassq_alloc(void); 329 extern void ifclassq_retain(struct ifclassq *); 330 extern void ifclassq_release(struct ifclassq **); 331 extern int ifclassq_setup_group(struct ifclassq *ifcq, uint8_t grp_idx, 332 uint8_t flags); 333 extern void ifclassq_set_grp_combined(struct ifclassq *ifcq, uint8_t grp_idx); 334 extern void ifclassq_set_grp_separated(struct ifclassq *ifcq, uint8_t grp_idx); 335 336 #endif /* BSD_KERNEL_PRIVATE */ 337 #endif /* PRIVATE */ 338 #endif /* _NET_CLASSQ_IF_CLASSQ_H_ */ 339