1 /*
2 * Copyright (c) 2018-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <kern/thread.h>
30 #include <kern/sched_prim.h>
31 #include <dev/random/randomdev.h>
32
33 #include <net/if.h>
34 #include <net/classq/classq.h>
35 #include <net/pktsched/pktsched.h>
36 #include <net/pktsched/pktsched_netem.h>
37
38 #if __LP64__
39 #define CONFIG_NETEM 1
40 #else
41 #define CONFIG_NETEM 0
42 #endif
43
44 #if CONFIG_NETEM
45
46 enum {
47 NETEM_LOG_ERROR = 0,
48 NETEM_LOG_INFO = 1,
49 NETEM_LOG_DEBUG = 2,
50 NETEM_LOG_HIDEBUG = 3,
51 };
52
53 #define NETEM_HEAP_SIZE 1024
54 #define NETEM_PSCALE IF_NETEM_PARAMS_PSCALE
55
56 #define netem_log(_level, _fmt, ...) \
57 do { \
58 if (pktsched_verbose > _level) { \
59 log(LOG_DEBUG, "NETEM: %-30s "_fmt "\n",\
60 __FUNCTION__, ##__VA_ARGS__); \
61 } \
62 } while (0);
63
64 extern kern_return_t thread_terminate(thread_t);
65
66 static LCK_GRP_DECLARE(netem_lock_group, "pktsched_netem_lock");
67
68 static const int32_t NORM_DIST_SCALE = 8192;
69 /* normal distribution lookup table */
70 static int32_t norm_dist_table[] =
71 {
72 -32768, -28307, -26871, -25967, -25298, -24765, -24320, -23937,
73 -23600, -23298, -23025, -22776, -22546, -22333, -22133, -21946,
74 -21770, -21604, -21445, -21295, -21151, -21013, -20882, -20755,
75 -20633, -20516, -20403, -20293, -20187, -20084, -19984, -19887,
76 -19793, -19702, -19612, -19526, -19441, -19358, -19277, -19198,
77 -19121, -19045, -18971, -18899, -18828, -18758, -18690, -18623,
78 -18557, -18492, -18429, -18366, -18305, -18245, -18185, -18127,
79 -18070, -18013, -17957, -17902, -17848, -17794, -17741, -17690,
80 -17638, -17588, -17538, -17489, -17440, -17392, -17345, -17298,
81 -17252, -17206, -17160, -17116, -17071, -17028, -16984, -16942,
82 -16899, -16857, -16816, -16775, -16735, -16694, -16654, -16615,
83 -16576, -16538, -16499, -16461, -16424, -16386, -16350, -16313,
84 -16277, -16241, -16205, -16170, -16135, -16100, -16066, -16031,
85 -15998, -15964, -15931, -15897, -15865, -15832, -15800, -15768,
86 -15736, -15704, -15673, -15642, -15611, -15580, -15550, -15519,
87 -15489, -15460, -15430, -15401, -15371, -15342, -15313, -15285,
88 -15256, -15228, -15200, -15172, -15144, -15116, -15089, -15062,
89 -15035, -15008, -14981, -14954, -14928, -14902, -14875, -14850,
90 -14823, -14798, -14772, -14747, -14722, -14696, -14671, -14647,
91 -14622, -14597, -14573, -14549, -14524, -14500, -14476, -14453,
92 -14429, -14405, -14382, -14359, -14335, -14312, -14289, -14266,
93 -14243, -14221, -14198, -14176, -14153, -14131, -14109, -14087,
94 -14065, -14043, -14021, -14000, -13978, -13957, -13935, -13914,
95 -13893, -13872, -13851, -13830, -13809, -13788, -13768, -13747,
96 -13727, -13706, -13686, -13666, -13646, -13626, -13606, -13586,
97 -13566, -13547, -13527, -13507, -13488, -13468, -13449, -13430,
98 -13411, -13392, -13373, -13354, -13335, -13316, -13297, -13278,
99 -13260, -13242, -13223, -13204, -13186, -13168, -13150, -13131,
100 -13113, -13095, -13077, -13060, -13042, -13024, -13006, -12988,
101 -12971, -12954, -12936, -12918, -12901, -12884, -12867, -12850,
102 -12832, -12815, -12798, -12781, -12764, -12748, -12731, -12714,
103 -12697, -12681, -12664, -12648, -12631, -12615, -12598, -12582,
104 -12566, -12549, -12533, -12517, -12501, -12485, -12469, -12453,
105 -12437, -12422, -12406, -12390, -12374, -12358, -12343, -12327,
106 -12312, -12296, -12281, -12265, -12250, -12235, -12220, -12204,
107 -12189, -12174, -12159, -12144, -12129, -12114, -12099, -12084,
108 -12069, -12054, -12039, -12025, -12010, -11995, -11981, -11966,
109 -11952, -11937, -11923, -11908, -11894, -11879, -11865, -11851,
110 -11837, -11822, -11808, -11794, -11780, -11766, -11752, -11737,
111 -11724, -11710, -11696, -11682, -11668, -11654, -11640, -11627,
112 -11613, -11599, -11586, -11572, -11559, -11545, -11531, -11518,
113 -11504, -11491, -11478, -11464, -11451, -11438, -11425, -11411,
114 -11398, -11385, -11372, -11359, -11346, -11332, -11319, -11306,
115 -11293, -11280, -11268, -11255, -11242, -11229, -11216, -11203,
116 -11191, -11178, -11165, -11153, -11140, -11127, -11114, -11102,
117 -11090, -11077, -11065, -11052, -11040, -11027, -11015, -11002,
118 -10990, -10978, -10965, -10953, -10941, -10929, -10917, -10904,
119 -10892, -10880, -10868, -10856, -10844, -10832, -10820, -10808,
120 -10796, -10784, -10772, -10760, -10748, -10736, -10725, -10713,
121 -10701, -10689, -10677, -10666, -10654, -10643, -10631, -10619,
122 -10607, -10596, -10584, -10573, -10562, -10550, -10539, -10527,
123 -10516, -10504, -10493, -10481, -10470, -10459, -10447, -10436,
124 -10425, -10414, -10402, -10391, -10380, -10369, -10358, -10346,
125 -10335, -10324, -10313, -10302, -10291, -10280, -10269, -10258,
126 -10247, -10236, -10225, -10214, -10203, -10192, -10181, -10171,
127 -10160, -10149, -10138, -10127, -10117, -10106, -10095, -10085,
128 -10074, -10063, -10052, -10042, -10031, -10021, -10010, -10000,
129 -9989, -9978, -9968, -9957, -9947, -9936, -9926, -9916,
130 -9905, -9895, -9884, -9874, -9864, -9853, -9843, -9833,
131 -9822, -9812, -9802, -9791, -9781, -9771, -9761, -9751,
132 -9741, -9730, -9720, -9710, -9700, -9690, -9680, -9670,
133 -9660, -9650, -9640, -9630, -9619, -9610, -9600, -9590,
134 -9580, -9570, -9560, -9550, -9540, -9530, -9520, -9511,
135 -9501, -9491, -9481, -9472, -9462, -9452, -9442, -9432,
136 -9423, -9413, -9403, -9394, -9384, -9374, -9365, -9355,
137 -9345, -9336, -9326, -9317, -9307, -9298, -9288, -9278,
138 -9269, -9259, -9250, -9241, -9231, -9221, -9212, -9202,
139 -9193, -9184, -9175, -9165, -9156, -9146, -9137, -9128,
140 -9119, -9109, -9100, -9090, -9081, -9072, -9063, -9053,
141 -9044, -9035, -9026, -9017, -9008, -8998, -8989, -8980,
142 -8971, -8962, -8953, -8944, -8934, -8925, -8916, -8907,
143 -8898, -8889, -8880, -8871, -8862, -8853, -8844, -8835,
144 -8826, -8817, -8808, -8799, -8790, -8781, -8772, -8764,
145 -8755, -8746, -8737, -8728, -8719, -8711, -8702, -8693,
146 -8684, -8675, -8667, -8658, -8649, -8640, -8632, -8623,
147 -8614, -8605, -8597, -8588, -8579, -8570, -8562, -8553,
148 -8545, -8536, -8527, -8519, -8510, -8502, -8493, -8484,
149 -8476, -8467, -8459, -8450, -8442, -8433, -8425, -8416,
150 -8408, -8399, -8391, -8382, -8374, -8365, -8357, -8348,
151 -8340, -8332, -8323, -8315, -8306, -8298, -8290, -8281,
152 -8273, -8264, -8256, -8248, -8240, -8231, -8223, -8215,
153 -8206, -8198, -8190, -8182, -8174, -8165, -8157, -8149,
154 -8140, -8132, -8124, -8116, -8108, -8099, -8091, -8083,
155 -8075, -8067, -8059, -8051, -8042, -8034, -8027, -8018,
156 -8010, -8002, -7994, -7986, -7978, -7970, -7962, -7954,
157 -7946, -7938, -7930, -7922, -7913, -7906, -7897, -7890,
158 -7882, -7874, -7866, -7858, -7850, -7842, -7834, -7826,
159 -7818, -7810, -7802, -7795, -7787, -7779, -7771, -7763,
160 -7755, -7748, -7739, -7732, -7724, -7716, -7708, -7700,
161 -7693, -7685, -7677, -7669, -7662, -7654, -7646, -7638,
162 -7630, -7623, -7615, -7608, -7600, -7592, -7584, -7577,
163 -7569, -7561, -7553, -7546, -7538, -7530, -7523, -7515,
164 -7508, -7500, -7492, -7485, -7477, -7469, -7462, -7454,
165 -7447, -7439, -7432, -7424, -7417, -7409, -7401, -7394,
166 -7386, -7379, -7372, -7364, -7356, -7349, -7341, -7334,
167 -7327, -7319, -7311, -7304, -7297, -7289, -7281, -7274,
168 -7267, -7259, -7252, -7245, -7237, -7230, -7222, -7215,
169 -7208, -7200, -7193, -7186, -7178, -7171, -7163, -7156,
170 -7149, -7141, -7134, -7127, -7119, -7112, -7105, -7098,
171 -7090, -7083, -7075, -7068, -7061, -7054, -7046, -7039,
172 -7032, -7025, -7018, -7010, -7003, -6996, -6989, -6981,
173 -6974, -6967, -6960, -6953, -6946, -6938, -6931, -6924,
174 -6917, -6910, -6903, -6895, -6888, -6881, -6874, -6867,
175 -6860, -6853, -6845, -6838, -6831, -6824, -6817, -6810,
176 -6803, -6796, -6789, -6782, -6775, -6767, -6760, -6753,
177 -6747, -6740, -6732, -6725, -6718, -6711, -6704, -6697,
178 -6690, -6683, -6676, -6669, -6662, -6655, -6648, -6641,
179 -6634, -6627, -6620, -6613, -6607, -6600, -6593, -6586,
180 -6579, -6572, -6565, -6558, -6551, -6544, -6538, -6531,
181 -6524, -6517, -6510, -6503, -6496, -6489, -6482, -6476,
182 -6469, -6462, -6455, -6448, -6441, -6434, -6428, -6421,
183 -6414, -6407, -6400, -6394, -6387, -6380, -6373, -6366,
184 -6360, -6353, -6346, -6339, -6333, -6326, -6319, -6312,
185 -6306, -6299, -6292, -6286, -6279, -6272, -6265, -6259,
186 -6252, -6245, -6239, -6232, -6225, -6219, -6212, -6205,
187 -6198, -6192, -6185, -6178, -6172, -6165, -6158, -6152,
188 -6145, -6139, -6132, -6125, -6119, -6112, -6105, -6099,
189 -6092, -6085, -6079, -6072, -6066, -6059, -6053, -6046,
190 -6040, -6033, -6026, -6019, -6013, -6006, -6000, -5993,
191 -5987, -5980, -5974, -5967, -5961, -5954, -5948, -5941,
192 -5935, -5928, -5922, -5915, -5908, -5902, -5895, -5889,
193 -5883, -5876, -5870, -5863, -5857, -5850, -5844, -5837,
194 -5831, -5825, -5818, -5811, -5805, -5799, -5792, -5786,
195 -5779, -5773, -5766, -5760, -5754, -5747, -5741, -5734,
196 -5728, -5722, -5715, -5709, -5702, -5696, -5690, -5683,
197 -5677, -5671, -5664, -5658, -5651, -5645, -5639, -5632,
198 -5626, -5620, -5613, -5607, -5600, -5594, -5588, -5582,
199 -5575, -5569, -5563, -5556, -5550, -5544, -5537, -5531,
200 -5525, -5519, -5512, -5506, -5500, -5494, -5487, -5481,
201 -5475, -5468, -5462, -5456, -5450, -5443, -5437, -5431,
202 -5425, -5418, -5412, -5406, -5400, -5393, -5387, -5381,
203 -5375, -5369, -5362, -5356, -5350, -5344, -5337, -5331,
204 -5325, -5319, -5313, -5306, -5300, -5294, -5288, -5282,
205 -5276, -5270, -5263, -5257, -5251, -5245, -5239, -5233,
206 -5226, -5220, -5214, -5208, -5202, -5196, -5190, -5183,
207 -5177, -5171, -5165, -5159, -5153, -5147, -5140, -5135,
208 -5129, -5122, -5116, -5110, -5104, -5098, -5092, -5086,
209 -5080, -5074, -5068, -5061, -5055, -5050, -5043, -5037,
210 -5031, -5025, -5019, -5013, -5007, -5001, -4995, -4989,
211 -4983, -4977, -4971, -4965, -4959, -4953, -4947, -4941,
212 -4935, -4929, -4923, -4917, -4911, -4905, -4899, -4893,
213 -4887, -4881, -4875, -4869, -4863, -4857, -4851, -4845,
214 -4839, -4833, -4827, -4821, -4815, -4809, -4803, -4797,
215 -4791, -4785, -4779, -4773, -4767, -4762, -4755, -4750,
216 -4744, -4738, -4732, -4726, -4720, -4714, -4708, -4702,
217 -4696, -4690, -4685, -4678, -4673, -4667, -4661, -4655,
218 -4649, -4643, -4637, -4631, -4626, -4620, -4614, -4608,
219 -4602, -4596, -4590, -4585, -4579, -4573, -4567, -4561,
220 -4555, -4549, -4544, -4538, -4532, -4526, -4520, -4514,
221 -4508, -4503, -4497, -4491, -4485, -4479, -4474, -4468,
222 -4462, -4456, -4450, -4445, -4439, -4433, -4427, -4421,
223 -4415, -4410, -4404, -4398, -4392, -4386, -4381, -4375,
224 -4369, -4363, -4358, -4352, -4346, -4340, -4334, -4329,
225 -4323, -4317, -4311, -4306, -4300, -4294, -4289, -4283,
226 -4277, -4271, -4266, -4260, -4254, -4248, -4243, -4237,
227 -4231, -4225, -4220, -4214, -4208, -4202, -4197, -4191,
228 -4185, -4180, -4174, -4168, -4162, -4157, -4151, -4146,
229 -4140, -4134, -4128, -4123, -4117, -4111, -4105, -4100,
230 -4094, -4089, -4083, -4077, -4071, -4066, -4060, -4055,
231 -4049, -4043, -4037, -4032, -4026, -4021, -4015, -4009,
232 -4003, -3998, -3992, -3987, -3981, -3975, -3970, -3964,
233 -3958, -3953, -3947, -3942, -3936, -3930, -3925, -3919,
234 -3913, -3908, -3902, -3897, -3891, -3885, -3880, -3874,
235 -3869, -3863, -3857, -3852, -3846, -3840, -3835, -3829,
236 -3824, -3818, -3813, -3807, -3801, -3796, -3790, -3785,
237 -3779, -3774, -3768, -3762, -3757, -3751, -3746, -3740,
238 -3734, -3729, -3723, -3718, -3712, -3707, -3701, -3696,
239 -3690, -3684, -3679, -3673, -3668, -3662, -3657, -3651,
240 -3646, -3640, -3635, -3629, -3624, -3618, -3613, -3607,
241 -3602, -3596, -3591, -3585, -3579, -3574, -3568, -3563,
242 -3557, -3552, -3546, -3541, -3535, -3530, -3524, -3519,
243 -3514, -3508, -3502, -3497, -3491, -3486, -3480, -3475,
244 -3469, -3464, -3459, -3453, -3448, -3442, -3437, -3431,
245 -3425, -3420, -3415, -3409, -3404, -3398, -3393, -3387,
246 -3382, -3376, -3371, -3366, -3360, -3355, -3349, -3344,
247 -3338, -3333, -3328, -3322, -3317, -3311, -3305, -3300,
248 -3295, -3289, -3284, -3278, -3273, -3268, -3262, -3257,
249 -3251, -3246, -3240, -3235, -3230, -3224, -3219, -3213,
250 -3208, -3203, -3197, -3192, -3186, -3181, -3176, -3170,
251 -3165, -3159, -3154, -3149, -3143, -3138, -3132, -3127,
252 -3122, -3116, -3111, -3105, -3100, -3095, -3089, -3084,
253 -3079, -3073, -3068, -3062, -3057, -3052, -3046, -3041,
254 -3036, -3030, -3025, -3019, -3014, -3009, -3003, -2998,
255 -2993, -2987, -2982, -2977, -2971, -2966, -2961, -2955,
256 -2950, -2944, -2939, -2934, -2928, -2923, -2918, -2912,
257 -2907, -2902, -2896, -2891, -2886, -2880, -2875, -2870,
258 -2864, -2859, -2854, -2848, -2843, -2838, -2832, -2827,
259 -2822, -2816, -2811, -2806, -2800, -2795, -2790, -2784,
260 -2779, -2774, -2768, -2763, -2758, -2753, -2747, -2742,
261 -2737, -2732, -2726, -2721, -2716, -2710, -2705, -2700,
262 -2694, -2689, -2684, -2678, -2673, -2668, -2663, -2657,
263 -2652, -2647, -2642, -2636, -2631, -2626, -2620, -2615,
264 -2610, -2605, -2599, -2594, -2589, -2583, -2578, -2573,
265 -2568, -2562, -2557, -2552, -2546, -2542, -2536, -2531,
266 -2526, -2520, -2515, -2510, -2505, -2499, -2494, -2489,
267 -2483, -2478, -2473, -2468, -2463, -2457, -2452, -2447,
268 -2442, -2436, -2431, -2426, -2421, -2415, -2410, -2405,
269 -2400, -2395, -2389, -2384, -2379, -2374, -2368, -2363,
270 -2358, -2353, -2347, -2342, -2337, -2332, -2327, -2321,
271 -2316, -2311, -2306, -2300, -2295, -2290, -2285, -2279,
272 -2275, -2269, -2264, -2259, -2254, -2248, -2243, -2238,
273 -2233, -2227, -2222, -2217, -2212, -2207, -2202, -2196,
274 -2191, -2186, -2181, -2175, -2170, -2165, -2160, -2155,
275 -2150, -2144, -2139, -2134, -2129, -2124, -2118, -2113,
276 -2108, -2103, -2098, -2093, -2087, -2082, -2077, -2072,
277 -2067, -2062, -2056, -2051, -2046, -2041, -2036, -2030,
278 -2025, -2020, -2015, -2010, -2005, -2000, -1994, -1989,
279 -1984, -1979, -1974, -1969, -1963, -1958, -1953, -1948,
280 -1943, -1937, -1932, -1927, -1922, -1917, -1912, -1907,
281 -1901, -1896, -1891, -1886, -1881, -1876, -1871, -1865,
282 -1860, -1855, -1850, -1845, -1840, -1835, -1829, -1824,
283 -1819, -1814, -1809, -1804, -1799, -1794, -1788, -1783,
284 -1778, -1773, -1768, -1763, -1758, -1752, -1747, -1742,
285 -1737, -1732, -1727, -1722, -1717, -1711, -1706, -1701,
286 -1696, -1691, -1686, -1681, -1676, -1670, -1665, -1660,
287 -1655, -1650, -1645, -1640, -1635, -1629, -1624, -1619,
288 -1614, -1609, -1604, -1599, -1594, -1589, -1584, -1579,
289 -1573, -1568, -1563, -1558, -1553, -1548, -1543, -1538,
290 -1532, -1527, -1522, -1517, -1512, -1507, -1502, -1497,
291 -1492, -1486, -1482, -1477, -1471, -1466, -1461, -1456,
292 -1451, -1446, -1441, -1436, -1431, -1425, -1420, -1415,
293 -1410, -1405, -1400, -1395, -1390, -1385, -1380, -1375,
294 -1370, -1364, -1359, -1354, -1349, -1344, -1339, -1334,
295 -1329, -1324, -1319, -1314, -1309, -1303, -1298, -1294,
296 -1288, -1283, -1278, -1273, -1268, -1263, -1258, -1253,
297 -1248, -1243, -1237, -1232, -1228, -1222, -1217, -1212,
298 -1207, -1202, -1197, -1192, -1187, -1182, -1177, -1171,
299 -1167, -1162, -1156, -1151, -1146, -1141, -1136, -1131,
300 -1126, -1121, -1116, -1111, -1106, -1101, -1096, -1091,
301 -1085, -1081, -1076, -1070, -1065, -1060, -1055, -1050,
302 -1045, -1040, -1035, -1030, -1025, -1020, -1015, -1010,
303 -1005, -1000, -995, -990, -985, -979, -974, -970,
304 -964, -959, -954, -949, -944, -939, -934, -929,
305 -924, -919, -914, -909, -904, -899, -894, -889,
306 -884, -879, -874, -868, -863, -859, -853, -848,
307 -843, -838, -833, -828, -823, -818, -813, -808,
308 -803, -798, -793, -788, -783, -778, -773, -768,
309 -763, -758, -752, -748, -743, -738, -732, -727,
310 -723, -717, -712, -707, -702, -697, -692, -687,
311 -682, -677, -672, -667, -662, -657, -652, -647,
312 -642, -637, -632, -627, -622, -617, -612, -607,
313 -602, -597, -591, -587, -582, -577, -571, -566,
314 -562, -557, -551, -546, -541, -537, -531, -526,
315 -521, -516, -511, -506, -501, -496, -491, -486,
316 -481, -476, -471, -466, -461, -456, -451, -446,
317 -441, -436, -431, -426, -421, -416, -411, -406,
318 -401, -396, -391, -386, -381, -376, -371, -366,
319 -360, -356, -351, -346, -340, -335, -331, -326,
320 -320, -315, -310, -306, -300, -295, -290, -285,
321 -281, -275, -270, -265, -261, -255, -250, -245,
322 -240, -235, -230, -225, -220, -215, -210, -205,
323 -200, -195, -190, -185, -180, -175, -170, -165,
324 -160, -155, -150, -145, -140, -135, -130, -125,
325 -120, -115, -110, -105, -100, -95, -90, -85,
326 -80, -75, -70, -65, -60, -55, -50, -45,
327 -40, -35, -29, -25, -20, -15, -9, -5,
328 0, 5, 11, 16, 20, 25, 30, 36,
329 41, 45, 50, 56, 61, 66, 70, 76,
330 81, 86, 91, 96, 101, 106, 111, 116,
331 121, 126, 131, 136, 141, 146, 151, 156,
332 161, 166, 171, 176, 181, 186, 191, 196,
333 201, 206, 211, 216, 221, 226, 231, 236,
334 241, 246, 251, 256, 261, 266, 271, 276,
335 281, 286, 291, 296, 301, 306, 311, 316,
336 322, 326, 331, 336, 342, 347, 351, 356,
337 362, 367, 372, 376, 382, 387, 392, 396,
338 402, 407, 412, 417, 422, 427, 432, 437,
339 442, 447, 452, 457, 462, 467, 472, 477,
340 482, 487, 492, 497, 502, 507, 512, 517,
341 522, 527, 532, 537, 542, 547, 552, 557,
342 562, 567, 572, 578, 582, 587, 593, 598,
343 603, 607, 613, 618, 623, 628, 633, 638,
344 643, 648, 653, 658, 663, 668, 673, 678,
345 683, 688, 693, 698, 703, 708, 713, 718,
346 723, 728, 733, 739, 743, 748, 754, 759,
347 763, 768, 774, 779, 784, 789, 794, 799,
348 804, 809, 814, 819, 824, 829, 834, 839,
349 844, 849, 854, 859, 864, 869, 874, 879,
350 884, 890, 895, 899, 905, 910, 915, 920,
351 925, 930, 935, 940, 945, 950, 955, 960,
352 965, 970, 975, 980, 985, 990, 995, 1001,
353 1006, 1010, 1016, 1021, 1026, 1031, 1036, 1041,
354 1046, 1051, 1056, 1061, 1066, 1071, 1076, 1081,
355 1086, 1092, 1096, 1102, 1107, 1112, 1117, 1122,
356 1127, 1132, 1137, 1142, 1147, 1152, 1157, 1162,
357 1167, 1173, 1178, 1183, 1188, 1193, 1198, 1203,
358 1208, 1213, 1218, 1223, 1228, 1233, 1238, 1244,
359 1248, 1254, 1259, 1264, 1269, 1274, 1279, 1284,
360 1289, 1294, 1299, 1304, 1309, 1314, 1320, 1325,
361 1330, 1335, 1340, 1345, 1350, 1355, 1360, 1365,
362 1371, 1375, 1381, 1386, 1391, 1396, 1401, 1406,
363 1411, 1416, 1421, 1426, 1432, 1436, 1442, 1447,
364 1452, 1457, 1462, 1467, 1472, 1477, 1482, 1488,
365 1493, 1497, 1503, 1508, 1513, 1518, 1523, 1528,
366 1534, 1538, 1543, 1549, 1554, 1559, 1564, 1569,
367 1574, 1579, 1584, 1590, 1595, 1600, 1605, 1610,
368 1615, 1620, 1625, 1630, 1636, 1640, 1646, 1651,
369 1656, 1661, 1666, 1671, 1676, 1681, 1687, 1692,
370 1697, 1702, 1707, 1712, 1717, 1722, 1728, 1733,
371 1738, 1743, 1748, 1753, 1758, 1764, 1769, 1774,
372 1779, 1784, 1789, 1794, 1799, 1805, 1810, 1815,
373 1820, 1825, 1831, 1835, 1841, 1846, 1851, 1856,
374 1861, 1866, 1871, 1877, 1882, 1887, 1892, 1897,
375 1902, 1908, 1913, 1918, 1923, 1928, 1933, 1939,
376 1944, 1949, 1954, 1959, 1964, 1969, 1975, 1980,
377 1985, 1990, 1995, 2000, 2005, 2011, 2016, 2021,
378 2026, 2031, 2037, 2042, 2047, 2052, 2057, 2062,
379 2068, 2073, 2078, 2083, 2088, 2093, 2099, 2104,
380 2109, 2114, 2119, 2125, 2130, 2135, 2140, 2145,
381 2150, 2156, 2161, 2166, 2171, 2177, 2182, 2187,
382 2192, 2197, 2202, 2208, 2213, 2218, 2223, 2229,
383 2234, 2239, 2244, 2249, 2254, 2260, 2265, 2270,
384 2275, 2281, 2286, 2291, 2296, 2302, 2306, 2312,
385 2317, 2322, 2327, 2333, 2338, 2343, 2348, 2354,
386 2359, 2364, 2369, 2374, 2380, 2385, 2390, 2395,
387 2401, 2406, 2411, 2416, 2422, 2427, 2432, 2437,
388 2442, 2448, 2453, 2458, 2463, 2469, 2474, 2479,
389 2485, 2490, 2495, 2500, 2506, 2511, 2516, 2521,
390 2526, 2532, 2537, 2542, 2548, 2553, 2558, 2563,
391 2569, 2574, 2579, 2585, 2589, 2595, 2600, 2605,
392 2611, 2616, 2621, 2627, 2632, 2637, 2642, 2648,
393 2653, 2658, 2664, 2669, 2674, 2680, 2685, 2690,
394 2695, 2700, 2706, 2711, 2716, 2722, 2727, 2732,
395 2738, 2743, 2748, 2754, 2759, 2764, 2769, 2775,
396 2780, 2785, 2791, 2796, 2801, 2807, 2812, 2817,
397 2823, 2828, 2833, 2839, 2844, 2849, 2855, 2860,
398 2865, 2870, 2876, 2881, 2886, 2892, 2897, 2902,
399 2908, 2913, 2918, 2924, 2929, 2935, 2940, 2945,
400 2951, 2956, 2961, 2967, 2972, 2977, 2983, 2988,
401 2993, 2999, 3004, 3010, 3015, 3020, 3026, 3031,
402 3036, 3042, 3047, 3052, 3058, 3063, 3069, 3074,
403 3079, 3085, 3090, 3095, 3101, 3106, 3112, 3117,
404 3122, 3128, 3133, 3139, 3144, 3149, 3155, 3160,
405 3166, 3171, 3176, 3182, 3187, 3193, 3198, 3203,
406 3209, 3214, 3220, 3225, 3231, 3236, 3242, 3247,
407 3252, 3258, 3263, 3269, 3274, 3279, 3285, 3290,
408 3296, 3301, 3307, 3312, 3317, 3323, 3328, 3334,
409 3339, 3345, 3350, 3355, 3361, 3367, 3372, 3378,
410 3383, 3388, 3394, 3399, 3405, 3410, 3416, 3421,
411 3427, 3432, 3437, 3443, 3448, 3454, 3459, 3465,
412 3471, 3476, 3481, 3487, 3492, 3498, 3503, 3509,
413 3514, 3520, 3525, 3531, 3536, 3542, 3548, 3553,
414 3558, 3564, 3569, 3575, 3580, 3586, 3591, 3597,
415 3602, 3608, 3613, 3619, 3625, 3630, 3636, 3641,
416 3647, 3652, 3658, 3663, 3669, 3675, 3680, 3686,
417 3691, 3697, 3702, 3708, 3713, 3719, 3724, 3730,
418 3736, 3741, 3747, 3752, 3758, 3763, 3769, 3774,
419 3780, 3786, 3791, 3797, 3802, 3808, 3813, 3819,
420 3825, 3830, 3836, 3842, 3847, 3853, 3858, 3864,
421 3869, 3875, 3881, 3886, 3892, 3898, 3903, 3909,
422 3915, 3920, 3926, 3931, 3937, 3942, 3948, 3954,
423 3960, 3965, 3971, 3976, 3982, 3987, 3993, 3999,
424 4005, 4010, 4016, 4021, 4027, 4033, 4039, 4044,
425 4050, 4055, 4061, 4067, 4073, 4078, 4084, 4089,
426 4095, 4101, 4107, 4112, 4118, 4123, 4129, 4135,
427 4141, 4146, 4152, 4158, 4164, 4169, 4175, 4181,
428 4187, 4192, 4198, 4203, 4209, 4215, 4221, 4226,
429 4232, 4238, 4243, 4249, 4255, 4261, 4266, 4272,
430 4278, 4284, 4289, 4295, 4301, 4307, 4313, 4318,
431 4324, 4330, 4336, 4341, 4347, 4353, 4359, 4364,
432 4370, 4376, 4382, 4388, 4393, 4399, 4405, 4411,
433 4417, 4422, 4428, 4434, 4440, 4445, 4452, 4457,
434 4463, 4469, 4474, 4481, 4486, 4492, 4498, 4504,
435 4510, 4515, 4521, 4527, 4533, 4539, 4545, 4551,
436 4556, 4562, 4568, 4574, 4580, 4585, 4592, 4597,
437 4603, 4609, 4615, 4621, 4627, 4633, 4638, 4644,
438 4650, 4656, 4662, 4668, 4674, 4680, 4686, 4692,
439 4697, 4703, 4709, 4715, 4721, 4727, 4733, 4739,
440 4745, 4751, 4757, 4762, 4769, 4774, 4780, 4786,
441 4792, 4798, 4804, 4810, 4816, 4822, 4828, 4834,
442 4840, 4846, 4852, 4858, 4864, 4870, 4876, 4882,
443 4888, 4894, 4900, 4906, 4912, 4918, 4924, 4930,
444 4936, 4942, 4948, 4954, 4960, 4966, 4972, 4978,
445 4984, 4990, 4996, 5002, 5008, 5014, 5020, 5026,
446 5032, 5038, 5045, 5050, 5057, 5063, 5069, 5075,
447 5081, 5087, 5093, 5099, 5105, 5111, 5118, 5123,
448 5129, 5136, 5142, 5148, 5154, 5160, 5166, 5172,
449 5179, 5185, 5191, 5197, 5203, 5209, 5215, 5221,
450 5227, 5233, 5240, 5246, 5252, 5258, 5265, 5271,
451 5277, 5283, 5289, 5295, 5301, 5308, 5314, 5320,
452 5326, 5333, 5339, 5345, 5351, 5357, 5363, 5369,
453 5376, 5382, 5388, 5394, 5401, 5407, 5413, 5419,
454 5426, 5432, 5438, 5444, 5451, 5457, 5463, 5469,
455 5476, 5482, 5488, 5494, 5501, 5507, 5513, 5520,
456 5526, 5532, 5539, 5545, 5551, 5557, 5564, 5570,
457 5576, 5583, 5589, 5596, 5602, 5608, 5614, 5621,
458 5627, 5634, 5640, 5646, 5652, 5659, 5665, 5672,
459 5678, 5684, 5691, 5697, 5704, 5710, 5716, 5723,
460 5729, 5736, 5742, 5748, 5755, 5761, 5768, 5774,
461 5780, 5787, 5793, 5800, 5806, 5813, 5819, 5826,
462 5832, 5838, 5845, 5852, 5858, 5864, 5871, 5877,
463 5884, 5890, 5897, 5903, 5910, 5916, 5923, 5929,
464 5936, 5942, 5949, 5956, 5962, 5968, 5975, 5981,
465 5988, 5994, 6001, 6008, 6014, 6021, 6027, 6034,
466 6041, 6047, 6054, 6060, 6067, 6074, 6080, 6087,
467 6093, 6100, 6107, 6113, 6120, 6126, 6133, 6140,
468 6146, 6153, 6160, 6167, 6173, 6180, 6186, 6193,
469 6200, 6206, 6213, 6220, 6226, 6233, 6240, 6246,
470 6253, 6260, 6266, 6273, 6280, 6287, 6294, 6300,
471 6307, 6314, 6321, 6327, 6334, 6341, 6348, 6354,
472 6361, 6368, 6375, 6382, 6388, 6395, 6402, 6409,
473 6416, 6422, 6429, 6436, 6443, 6450, 6457, 6463,
474 6470, 6477, 6484, 6491, 6497, 6504, 6511, 6518,
475 6525, 6532, 6539, 6546, 6553, 6559, 6566, 6573,
476 6580, 6587, 6594, 6601, 6608, 6615, 6622, 6629,
477 6636, 6643, 6650, 6657, 6664, 6671, 6678, 6685,
478 6692, 6699, 6706, 6713, 6719, 6727, 6734, 6741,
479 6748, 6755, 6762, 6769, 6776, 6783, 6790, 6797,
480 6804, 6811, 6818, 6826, 6833, 6840, 6847, 6854,
481 6861, 6868, 6875, 6883, 6889, 6897, 6904, 6911,
482 6918, 6925, 6932, 6939, 6947, 6954, 6961, 6969,
483 6975, 6983, 6990, 6997, 7005, 7012, 7019, 7026,
484 7033, 7041, 7048, 7055, 7062, 7070, 7077, 7084,
485 7091, 7099, 7106, 7114, 7121, 7128, 7135, 7143,
486 7150, 7157, 7165, 7172, 7179, 7187, 7194, 7202,
487 7209, 7216, 7224, 7231, 7238, 7246, 7253, 7261,
488 7268, 7276, 7283, 7290, 7298, 7306, 7313, 7320,
489 7328, 7336, 7343, 7350, 7358, 7365, 7373, 7381,
490 7388, 7395, 7403, 7410, 7418, 7426, 7433, 7441,
491 7448, 7456, 7463, 7471, 7479, 7486, 7494, 7501,
492 7509, 7517, 7524, 7532, 7540, 7547, 7555, 7563,
493 7571, 7578, 7586, 7594, 7601, 7609, 7617, 7624,
494 7632, 7640, 7648, 7655, 7663, 7671, 7679, 7687,
495 7694, 7702, 7710, 7718, 7725, 7733, 7741, 7749,
496 7757, 7765, 7773, 7780, 7788, 7796, 7804, 7812,
497 7820, 7828, 7836, 7843, 7852, 7859, 7868, 7875,
498 7883, 7891, 7899, 7907, 7915, 7923, 7931, 7939,
499 7947, 7955, 7963, 7971, 7979, 7988, 7995, 8004,
500 8012, 8020, 8028, 8036, 8044, 8052, 8061, 8069,
501 8076, 8085, 8093, 8101, 8109, 8117, 8126, 8134,
502 8142, 8150, 8158, 8167, 8175, 8183, 8192, 8200,
503 8208, 8217, 8225, 8233, 8241, 8250, 8258, 8266,
504 8275, 8283, 8292, 8300, 8308, 8317, 8325, 8333,
505 8342, 8350, 8359, 8367, 8376, 8384, 8392, 8401,
506 8409, 8418, 8426, 8435, 8443, 8452, 8461, 8469,
507 8477, 8486, 8495, 8503, 8512, 8520, 8529, 8538,
508 8546, 8555, 8564, 8573, 8581, 8590, 8598, 8607,
509 8616, 8625, 8633, 8642, 8651, 8659, 8668, 8677,
510 8686, 8695, 8704, 8712, 8721, 8730, 8739, 8748,
511 8756, 8765, 8774, 8783, 8792, 8801, 8810, 8819,
512 8828, 8837, 8846, 8855, 8864, 8873, 8882, 8891,
513 8900, 8909, 8918, 8927, 8936, 8945, 8954, 8964,
514 8973, 8982, 8991, 9000, 9009, 9019, 9028, 9037,
515 9046, 9055, 9064, 9074, 9083, 9092, 9102, 9111,
516 9120, 9130, 9139, 9148, 9157, 9167, 9176, 9186,
517 9195, 9205, 9214, 9223, 9233, 9242, 9252, 9261,
518 9271, 9280, 9290, 9300, 9309, 9318, 9328, 9338,
519 9347, 9357, 9367, 9376, 9386, 9395, 9405, 9415,
520 9424, 9434, 9444, 9454, 9464, 9473, 9483, 9493,
521 9503, 9513, 9522, 9532, 9542, 9552, 9562, 9572,
522 9582, 9592, 9602, 9612, 9622, 9632, 9642, 9652,
523 9662, 9672, 9682, 9692, 9702, 9712, 9722, 9733,
524 9743, 9753, 9763, 9773, 9783, 9794, 9804, 9814,
525 9825, 9835, 9845, 9855, 9866, 9876, 9887, 9897,
526 9907, 9918, 9928, 9939, 9949, 9960, 9970, 9981,
527 9991, 10002, 10012, 10023, 10034, 10044, 10055, 10066,
528 10076, 10087, 10097, 10108, 10119, 10130, 10140, 10152,
529 10162, 10173, 10184, 10195, 10206, 10217, 10227, 10238,
530 10249, 10260, 10271, 10282, 10293, 10304, 10315, 10326,
531 10337, 10349, 10360, 10371, 10382, 10394, 10405, 10416,
532 10427, 10438, 10450, 10461, 10472, 10484, 10495, 10507,
533 10518, 10530, 10541, 10553, 10564, 10575, 10587, 10598,
534 10610, 10622, 10633, 10645, 10657, 10668, 10680, 10692,
535 10704, 10715, 10727, 10739, 10751, 10763, 10775, 10786,
536 10798, 10811, 10822, 10834, 10847, 10858, 10870, 10883,
537 10895, 10907, 10919, 10931, 10944, 10956, 10968, 10981,
538 10993, 11005, 11017, 11030, 11042, 11055, 11067, 11080,
539 11092, 11105, 11117, 11130, 11142, 11155, 11168, 11180,
540 11193, 11206, 11219, 11232, 11245, 11257, 11270, 11283,
541 11296, 11309, 11322, 11335, 11348, 11361, 11375, 11388,
542 11401, 11414, 11427, 11441, 11454, 11467, 11481, 11494,
543 11508, 11521, 11534, 11548, 11561, 11575, 11589, 11602,
544 11616, 11630, 11644, 11657, 11671, 11685, 11699, 11713,
545 11727, 11741, 11755, 11769, 11783, 11797, 11811, 11826,
546 11839, 11854, 11868, 11882, 11897, 11911, 11926, 11940,
547 11955, 11969, 11984, 11998, 12013, 12028, 12043, 12057,
548 12072, 12087, 12102, 12117, 12132, 12147, 12162, 12177,
549 12193, 12208, 12223, 12238, 12254, 12269, 12284, 12299,
550 12315, 12331, 12346, 12362, 12378, 12393, 12409, 12425,
551 12441, 12457, 12473, 12489, 12505, 12521, 12537, 12553,
552 12569, 12586, 12602, 12619, 12635, 12651, 12668, 12684,
553 12701, 12718, 12734, 12751, 12768, 12785, 12802, 12819,
554 12836, 12853, 12870, 12888, 12905, 12922, 12940, 12957,
555 12975, 12993, 13010, 13028, 13046, 13064, 13081, 13099,
556 13117, 13135, 13154, 13172, 13190, 13209, 13227, 13246,
557 13264, 13283, 13301, 13320, 13339, 13358, 13377, 13396,
558 13415, 13434, 13454, 13473, 13492, 13512, 13532, 13551,
559 13571, 13591, 13611, 13631, 13651, 13671, 13691, 13711,
560 13732, 13752, 13773, 13793, 13814, 13835, 13856, 13877,
561 13898, 13919, 13940, 13962, 13983, 14005, 14026, 14048,
562 14070, 14092, 14114, 14136, 14159, 14181, 14203, 14226,
563 14249, 14272, 14294, 14318, 14341, 14364, 14387, 14411,
564 14434, 14458, 14482, 14506, 14530, 14554, 14578, 14603,
565 14628, 14653, 14677, 14703, 14728, 14753, 14778, 14804,
566 14830, 14855, 14882, 14908, 14934, 14961, 14987, 15014,
567 15041, 15068, 15095, 15123, 15151, 15179, 15206, 15235,
568 15263, 15291, 15320, 15349, 15378, 15408, 15437, 15466,
569 15496, 15527, 15557, 15587, 15618, 15649, 15680, 15712,
570 15743, 15775, 15808, 15840, 15872, 15906, 15939, 15972,
571 16006, 16040, 16074, 16108, 16143, 16178, 16214, 16249,
572 16285, 16322, 16358, 16395, 16433, 16470, 16508, 16547,
573 16586, 16624, 16664, 16704, 16744, 16785, 16826, 16867,
574 16910, 16952, 16995, 17038, 17082, 17126, 17171, 17217,
575 17263, 17309, 17356, 17403, 17452, 17501, 17550, 17600,
576 17651, 17702, 17754, 17807, 17861, 17915, 17970, 18026,
577 18083, 18141, 18200, 18259, 18320, 18382, 18444, 18508,
578 18573, 18639, 18706, 18775, 18845, 18917, 18989, 19064,
579 19140, 19217, 19297, 19378, 19461, 19547, 19634, 19724,
580 19816, 19911, 20009, 20109, 20213, 20319, 20430, 20544,
581 20663, 20786, 20914, 21047, 21186, 21331, 21484, 21644,
582 21813, 21991, 22181, 22384, 22601, 22836, 23091, 23370,
583 23679, 24027, 24424, 24888, 25450, 26164, 27159, 28858,
584 };
585 #define NORM_DIST_TABLE_SIZE \
586 (sizeof (norm_dist_table) / sizeof (norm_dist_table[0]))
587
588 struct heap_elem {
589 uint64_t key;
590 pktsched_pkt_t pkt;
591 };
592
593 struct heap {
594 uint64_t limit; /* max size */
595 uint64_t size; /* current size */
596 struct heap_elem p[0];
597 };
598
599 static struct heap *heap_create(uint64_t size);
600 static int heap_insert(struct heap *h, uint64_t k, pktsched_pkt_t *p);
601 static int heap_peek(struct heap *h, uint64_t *k, pktsched_pkt_t *p);
602 static int heap_extract(struct heap *h, uint64_t *k, pktsched_pkt_t *p);
603
604 struct netem {
605 decl_lck_mtx_data(, netem_lock);
606
607 /* Init Time Constants */
608 char netem_name[MAXTHREADNAMESIZE];
609 uint32_t netem_flags;
610 struct thread *netem_output_thread;
611
612 void *netem_output_handle;
613 int (*netem_output)(void *handle, pktsched_pkt_t *pkts,
614 uint32_t n_pkts);
615 uint32_t netem_output_max_batch_size;
616
617 struct heap *netem_heap;
618
619 /* Parameters variables */
620 /* bandwidth token bucket limit */
621 #define TOKEN_INVALID UINT64_MAX
622 struct token_bucket {
623 uint64_t depth;
624 uint64_t token;
625 uint64_t last;
626 uint64_t rate;
627 } netem_bandwidth_model;
628
629 /* XXX (need correlated) naive corruption model */
630 struct corruption {
631 uint32_t corruption_p;
632 } netem_corruption_model;
633
634 /* naive duplication model */
635 struct duplication {
636 uint32_t duplication_p;
637 } netem_duplication_model;
638
639 /* latency (with jitter following random distribution) */
640 struct latency {
641 uint32_t latency_ms;
642 uint32_t jitter_ms;
643 uint64_t last_time_to_send;
644 } netem_latency_model;
645
646 /* 4 state Markov packet loss model */
647 struct loss {
648 enum _4state_markov_packet_loss_state {
649 __NO_LOSS = 0,
650 GAP_RX = 1,
651 GAP_LOSS,
652 BURST_RX,
653 BURST_LOSS,
654 } state;
655
656 uint32_t p_gr_gl; /* P( gap_loss | gap_rx ) */
657 uint32_t p_gr_bl; /* P( burst_loss | gap_rx ) */
658 uint32_t p_bl_br; /* P( burst_rx | burst_loss ) */
659 uint32_t p_bl_gr; /* P( gap_rx | burst_loss ) */
660 uint32_t p_br_bl; /* P( burst_loss | burst_rx ) */
661 } netem_loss_model;
662
663 /*
664 * Reordering Model --
665 * randomly select packets and re-inject with additional delay
666 */
667 struct reordering {
668 uint32_t reordering_p;
669 } netem_reordering_model;
670 };
671
672 #define NETEMF_INITIALIZED 0x00000001 /* has been initialized */
673 #define NETEMF_RUNNING 0x00000002 /* thread is running */
674 #define NETEMF_TERMINATEBLOCK 0x20000000 /* block waiting terminate */
675 #define NETEMF_TERMINATING 0x40000000 /* thread is terminating */
676 #define NETEMF_TERMINATED 0x80000000 /* thread is terminated */
677
678 #define NETEM_MTX_LOCK(_sch) \
679 lck_mtx_lock(&(_sch)->netem_lock)
680 #define NETEM_MTX_LOCK_ASSERT_HELD(_sch) \
681 LCK_MTX_ASSERT(&(_sch)->netem_lock, LCK_ASSERT_OWNED)
682 #define NETEM_MTX_LOCK_ASSERT_NOTHELD(_sch) \
683 LCK_MTX_ASSERT(&(_sch)->netem_lock, LCK_ASSERT_NOTOWNED)
684 #define NETEM_MTX_UNLOCK(_sch) \
685 lck_mtx_unlock(&(_sch)->netem_lock)
686
687 static struct heap *
heap_create(uint64_t limit)688 heap_create(uint64_t limit)
689 {
690 struct heap *h = NULL;
691
692 // verify limit
693 size_t size = sizeof(struct heap) + sizeof(struct heap_elem) * limit;
694
695 h = (struct heap *)kalloc_data(size, Z_WAITOK | Z_ZERO);
696 if (h == NULL) {
697 return NULL;
698 }
699
700 h->limit = limit;
701 h->size = 0;
702
703 return h;
704 }
705
706 static void
heap_destroy(struct heap * h)707 heap_destroy(struct heap *h)
708 {
709 ASSERT(h->size == 0);
710
711 kfree_data(h, sizeof(struct heap) + sizeof(struct heap_elem) * h->limit);
712 }
713
714 #define HEAP_FATHER(child) (((child) - 1) / 2)
715 #define HEAP_SWAP(a, b, tmp) { tmp = a; a = b; b = tmp; }
716 #define HEAP_LEFT(x) (2 * (x) + 1)
717
718 static int
heap_insert(struct heap * h,uint64_t key,pktsched_pkt_t * pkt)719 heap_insert(struct heap *h, uint64_t key, pktsched_pkt_t *pkt)
720 {
721 ASSERT(h != NULL);
722
723 if (h->size == h->limit) {
724 return ENOBUFS;
725 }
726
727 uint64_t child, parent;
728 if (pkt == NULL) {
729 child = key;
730 ASSERT(child < h->size);
731 } else {
732 child = h->size;
733 h->p[child].key = key;
734 h->p[child].pkt = *pkt;
735 h->size++;
736 }
737
738 while (child > 0) {
739 struct heap_elem tmp;
740 parent = HEAP_FATHER(child);
741 if (h->p[parent].key < h->p[child].key) {
742 break;
743 }
744 HEAP_SWAP(h->p[child], h->p[parent], tmp);
745 child = parent;
746 }
747
748 return 0;
749 }
750
751 static int
heap_peek(struct heap * h,uint64_t * key,pktsched_pkt_t * pkt)752 heap_peek(struct heap *h, uint64_t *key, pktsched_pkt_t *pkt)
753 {
754 if (h->size == 0) {
755 return ENOENT;
756 }
757
758 *key = h->p[0].key;
759 *pkt = h->p[0].pkt;
760 return 0;
761 }
762
763 static int
heap_extract(struct heap * h,uint64_t * key,pktsched_pkt_t * pkt)764 heap_extract(struct heap *h, uint64_t *key, pktsched_pkt_t *pkt)
765 {
766 uint64_t child, parent, max;
767
768 if (h->size == 0) {
769 netem_log(NETEM_LOG_ERROR, "warning: extract from empty heap");
770 return ENOENT;
771 }
772
773 *key = h->p[0].key;
774 *pkt = h->p[0].pkt;
775
776 /* re-heapify */
777 parent = 0;
778 child = HEAP_LEFT(parent); /* start from left child */
779 max = h->size - 1;
780 while (child <= max) {
781 if (child != max && h->p[child + 1].key < h->p[child].key) {
782 child = child + 1; /* right child */
783 }
784 h->p[parent] = h->p[child];
785 parent = child;
786 child = HEAP_LEFT(child); /* left child for next loop */
787 }
788
789 h->size--;
790 if (parent != max) {
791 /* Fill hole with last entry, bubble up reusing insert code */
792 h->p[parent] = h->p[max];
793 _PKTSCHED_PKT_INIT(&h->p[max].pkt);
794 heap_insert(h, parent, NULL); /* this one cannot fail */
795 }
796
797 return 0;
798 }
799
800 static void
token_bucket_update(struct token_bucket * tb)801 token_bucket_update(struct token_bucket *tb)
802 {
803 uint64_t now, elapsed;
804 clock_sec_t sec;
805 clock_usec_t usec;
806
807 if (tb->rate == 0) {
808 return;
809 }
810
811 now = mach_absolute_time();
812 elapsed = now - tb->last;
813 absolutetime_to_microtime(elapsed, &sec, &usec);
814 tb->token += ((sec * USEC_PER_SEC + usec) * tb->rate / USEC_PER_SEC);
815 if (__improbable(tb->token > tb->depth)) {
816 tb->token = tb->depth;
817 }
818 tb->last = now;
819 }
820
821 static boolean_t
bandwidth_limited(struct netem * ne,uint32_t pkt_len)822 bandwidth_limited(struct netem *ne, uint32_t pkt_len)
823 {
824 struct token_bucket *tb = &ne->netem_bandwidth_model;
825
826 if (tb->rate == 0) {
827 return FALSE;
828 }
829
830 if (tb->token < pkt_len * 8) {
831 netem_log(NETEM_LOG_DEBUG, "limited");
832 return TRUE;
833 }
834 tb->token -= pkt_len * 8;
835
836 netem_log(NETEM_LOG_DEBUG, "token left %llu", tb->token);
837
838 return FALSE;
839 }
840
841 static void
corruption_event(struct netem * ne,pktsched_pkt_t * pkt)842 corruption_event(struct netem *ne, pktsched_pkt_t *pkt)
843 {
844 struct corruption *corr = &ne->netem_corruption_model;
845 uint32_t rand;
846
847 if (corr->corruption_p == 0) {
848 return;
849 }
850
851 read_frandom(&rand, sizeof(rand));
852 rand %= NETEM_PSCALE;
853
854 if (rand < corr->corruption_p) {
855 netem_log(NETEM_LOG_ERROR, "\t corrupted");
856 pktsched_corrupt_packet(pkt);
857 }
858 }
859
860 static boolean_t
duplication_event(struct netem * ne)861 duplication_event(struct netem *ne)
862 {
863 struct duplication *dup = &ne->netem_duplication_model;
864 uint32_t rand;
865
866 if (dup->duplication_p == 0) {
867 return FALSE;
868 }
869
870 read_frandom(&rand, sizeof(rand));
871 rand %= NETEM_PSCALE;
872
873 return rand < dup->duplication_p;
874 }
875
876 static uint64_t
latency_event(struct netem * ne,boolean_t reordering)877 latency_event(struct netem *ne, boolean_t reordering)
878 {
879 struct latency *l = &ne->netem_latency_model;
880 int32_t delay_ms = 0, jitter_ms = 0;
881 uint64_t time_to_send = 0;
882
883 delay_ms = l->latency_ms;
884 if (l->jitter_ms != 0) {
885 int32_t rand, x, t, s = l->jitter_ms;
886 read_frandom(&rand, sizeof(rand));
887 t = norm_dist_table[rand % NORM_DIST_TABLE_SIZE];
888 x = (s % NORM_DIST_SCALE) * t;
889 if (x >= 0) {
890 x += NORM_DIST_SCALE / 2;
891 } else {
892 x -= NORM_DIST_SCALE / 2;
893 }
894 jitter_ms = x / NORM_DIST_SCALE + (s * t / NORM_DIST_SCALE);
895 }
896
897 delay_ms += jitter_ms;
898 delay_ms = MAX(delay_ms, 0);
899
900 netem_log(NETEM_LOG_DEBUG, "\tdelay %dms", delay_ms);
901 clock_interval_to_deadline(delay_ms, NSEC_PER_MSEC, &time_to_send);
902
903 if (l->last_time_to_send != 0) {
904 if (reordering) {
905 /* reorder with last packet */
906 time_to_send = l->last_time_to_send - 1;
907 } else {
908 /* make sure packet time to send is monotonic */
909 if (time_to_send < l->last_time_to_send) {
910 /* send this one immediately afterwards */
911 time_to_send = l->last_time_to_send + 1;
912 }
913 }
914 }
915
916 l->last_time_to_send = time_to_send;
917
918 return time_to_send;
919 }
920
921 static boolean_t
loss_event(struct netem * ne)922 loss_event(struct netem *ne)
923 {
924 struct loss *loss = &ne->netem_loss_model;
925 uint32_t rand;
926
927 if (loss->state == __NO_LOSS) {
928 return FALSE;
929 }
930
931 read_frandom(&rand, sizeof(rand));
932 rand %= NETEM_PSCALE;
933
934 switch (loss->state) {
935 case GAP_RX:
936 if (rand < loss->p_gr_gl) {
937 loss->state = GAP_RX;
938 return TRUE;
939 } else if (loss->p_gr_gl < rand &&
940 rand < loss->p_gr_gl + loss->p_gr_bl) {
941 loss->state = BURST_LOSS;
942 return TRUE;
943 } else {
944 loss->state = GAP_RX;
945 return FALSE;
946 }
947 case BURST_LOSS:
948 if (rand < loss->p_bl_br) {
949 loss->state = BURST_RX;
950 return FALSE;
951 } else if (loss->p_bl_br < rand &&
952 rand < loss->p_bl_br + loss->p_bl_gr) {
953 loss->state = GAP_RX;
954 return FALSE;
955 } else {
956 loss->state = BURST_LOSS;
957 return TRUE;
958 }
959 case BURST_RX:
960 if (rand < loss->p_br_bl) {
961 loss->state = BURST_LOSS;
962 return TRUE;
963 } else {
964 loss->state = BURST_RX;
965 return FALSE;
966 }
967 case GAP_LOSS:
968 /* This is instantaneous (stateless), should not be reached */
969 default:
970 VERIFY(0);
971 break;
972 }
973
974 /* not reached */
975 VERIFY(0);
976 return FALSE;
977 }
978
979 static boolean_t
reordering_event(struct netem * ne)980 reordering_event(struct netem *ne)
981 {
982 struct reordering *reord = &ne->netem_reordering_model;
983 uint32_t rand;
984
985 if (reord->reordering_p == 0) {
986 return FALSE;
987 }
988
989 read_frandom(&rand, sizeof(rand));
990 rand %= NETEM_PSCALE;
991
992 return rand < reord->reordering_p;
993 }
994
995 static void
netem_update_locked(struct netem * ne)996 netem_update_locked(struct netem *ne)
997 {
998 ASSERT(ne != NULL);
999 NETEM_MTX_LOCK_ASSERT_HELD(ne);
1000
1001 token_bucket_update(&ne->netem_bandwidth_model);
1002 }
1003
1004 int
netem_enqueue(struct netem * ne,classq_pkt_t * p,boolean_t * pdrop)1005 netem_enqueue(struct netem *ne, classq_pkt_t *p, boolean_t *pdrop)
1006 {
1007 int ret = 0;
1008 int pkt_count = 1;
1009 uint64_t time_to_send;
1010 pktsched_pkt_t pkt;
1011
1012 pktsched_pkt_encap(&pkt, p);
1013
1014 ASSERT(ne != NULL);
1015 ASSERT(pdrop != NULL);
1016 NETEM_MTX_LOCK(ne);
1017
1018 netem_log(NETEM_LOG_DEBUG, "+ %p begin", p->cp_mbuf);
1019
1020 if (loss_event(ne)) {
1021 netem_log(NETEM_LOG_DEBUG, "\t lost");
1022 pkt_count--;
1023 }
1024
1025 if (duplication_event(ne)) {
1026 netem_log(NETEM_LOG_DEBUG, "\t dup'ed");
1027 pkt_count++;
1028 }
1029
1030 if (pkt_count == 0) {
1031 pktsched_free_pkt(&pkt);
1032 *pdrop = TRUE;
1033 goto done;
1034 }
1035
1036 do {
1037 corruption_event(ne, &pkt);
1038
1039 time_to_send = latency_event(ne, reordering_event(ne));
1040
1041 ret = heap_insert(ne->netem_heap, time_to_send, &pkt);
1042 if (ret != 0) {
1043 netem_log(NETEM_LOG_DEBUG, "\t%p err heap_insert %d",
1044 p->cp_mbuf, ret);
1045 pktsched_free_pkt(&pkt);
1046 goto done;
1047 }
1048 netem_log(NETEM_LOG_DEBUG, "\t%p enqueued",
1049 pkt.pktsched_pkt_mbuf);
1050 } while (--pkt_count > 0 &&
1051 __probable((ret = pktsched_clone_pkt(&pkt, &pkt)) == 0));
1052
1053 done:
1054 if (__probable(ne->netem_output_thread != THREAD_NULL)) {
1055 if (!(ne->netem_flags & (NETEMF_RUNNING |
1056 NETEMF_TERMINATING | NETEMF_TERMINATED))) {
1057 netem_log(NETEM_LOG_DEBUG, "wakeup output thread");
1058 (void) thread_wakeup((caddr_t)&ne->netem_flags);
1059 }
1060 }
1061
1062 NETEM_MTX_UNLOCK(ne);
1063 netem_log(NETEM_LOG_DEBUG, "- %p end", p->cp_mbuf);
1064
1065 return ret;
1066 }
1067
1068 static int
netem_dequeue_internal_locked(struct netem * ne,pktsched_pkt_t * pp,boolean_t * ppending)1069 netem_dequeue_internal_locked(struct netem *ne, pktsched_pkt_t *pp,
1070 boolean_t *ppending)
1071 {
1072 int ret = 0;
1073 uint64_t time_to_send;
1074 pktsched_pkt_t pkt;
1075
1076 ASSERT(ne != NULL);
1077 NETEM_MTX_LOCK_ASSERT_HELD(ne);
1078
1079 netem_log(NETEM_LOG_HIDEBUG, "+ begin");
1080
1081 ret = heap_peek(ne->netem_heap, &time_to_send, &pkt);
1082 if (ret != 0) {
1083 netem_log(NETEM_LOG_HIDEBUG, "\theap empty");
1084 ret = ENOENT;
1085 goto done;
1086 }
1087
1088 /* latency limit */
1089 if (time_to_send > mach_absolute_time()) {
1090 netem_log(NETEM_LOG_DEBUG,
1091 "held back: time_to_send %llu now %llu",
1092 time_to_send, mach_absolute_time());
1093 ret = EAGAIN;
1094 goto done;
1095 }
1096
1097 /* bandwidth limited */
1098 if (bandwidth_limited(ne, pkt.pktsched_plen)) {
1099 ret = EAGAIN;
1100 goto done;
1101 }
1102
1103 ret = heap_extract(ne->netem_heap, &time_to_send, &pkt);
1104 ASSERT(ret == 0);
1105 *pp = pkt;
1106
1107 netem_log(NETEM_LOG_HIDEBUG, "- %p end", pkt.pktsched_pkt_mbuf);
1108
1109 done:
1110 *ppending = (ret == EAGAIN) ? TRUE : FALSE;
1111
1112 return ret;
1113 }
1114
1115 __attribute__((noreturn))
1116 static void
netem_output_thread_cont(void * v,wait_result_t w)1117 netem_output_thread_cont(void *v, wait_result_t w)
1118 __attribute__((optnone))
1119 {
1120 struct netem *ne = v;
1121 boolean_t pending = FALSE;
1122 pktsched_pkt_t pkts[NETEM_MAX_BATCH_SIZE];
1123 uint32_t n_pkts = 0;
1124 int ret;
1125
1126 NETEM_MTX_LOCK(ne);
1127 ASSERT(!(ne->netem_flags & NETEMF_TERMINATED));
1128 ne->netem_flags |= NETEMF_RUNNING;
1129
1130 if (__improbable(w == THREAD_INTERRUPTED ||
1131 (ne->netem_flags & NETEMF_TERMINATING) != 0)) {
1132 ASSERT(!(ne->netem_flags & NETEMF_TERMINATED));
1133 ne->netem_flags &= ~(NETEMF_RUNNING | NETEMF_TERMINATING);
1134 ne->netem_flags |= NETEMF_TERMINATED;
1135
1136 netem_log(NETEM_LOG_INFO, "%s output thread terminated",
1137 ne->netem_name);
1138
1139 if (ne->netem_flags & NETEMF_TERMINATEBLOCK) {
1140 thread_wakeup((caddr_t)&ne->netem_output_thread);
1141 }
1142
1143 NETEM_MTX_UNLOCK(ne);
1144
1145 /* for the extra refcnt from kernel_thread_start() */
1146 thread_deallocate(current_thread());
1147 /* this is the end */
1148 thread_terminate(current_thread());
1149 /* NOTREACHED */
1150 __builtin_unreachable();
1151 }
1152
1153 ASSERT(ne->netem_output != NULL);
1154 netem_update_locked(ne);
1155 n_pkts = 0;
1156 for (;;) {
1157 ret = netem_dequeue_internal_locked(ne, &pkts[n_pkts],
1158 &pending);
1159 if (__probable(ret == 0 &&
1160 ++n_pkts < ne->netem_output_max_batch_size)) {
1161 continue;
1162 }
1163
1164 if (__probable(n_pkts != 0)) {
1165 NETEM_MTX_UNLOCK(ne);
1166 (void) ne->netem_output(ne->netem_output_handle,
1167 pkts, n_pkts);
1168 NETEM_MTX_LOCK(ne);
1169 n_pkts = 0;
1170 }
1171 if (ret != 0) {
1172 break;
1173 }
1174 }
1175
1176 uint64_t deadline = TIMEOUT_WAIT_FOREVER;
1177 if (pending) {
1178 clock_interval_to_deadline(1, NSEC_PER_MSEC, &deadline);
1179 }
1180 (void) assert_wait_deadline(&ne->netem_flags, THREAD_UNINT, deadline);
1181 ne->netem_flags &= ~NETEMF_RUNNING;
1182 NETEM_MTX_UNLOCK(ne);
1183 (void) thread_block_parameter(netem_output_thread_cont, ne);
1184 /* NOTREACHED */
1185 __builtin_unreachable();
1186 }
1187
1188 __attribute__((noreturn))
1189 static void
netem_output_thread_func(void * v,wait_result_t w)1190 netem_output_thread_func(void *v, wait_result_t w)
1191 {
1192 #pragma unused(w)
1193 struct netem *ne = v;
1194
1195 ASSERT(ne->netem_output_thread == current_thread());
1196 thread_set_thread_name(current_thread(), ne->netem_name);
1197
1198 NETEM_MTX_LOCK(ne);
1199 VERIFY(!(ne->netem_flags & NETEMF_RUNNING));
1200 (void) assert_wait(&ne->netem_flags, THREAD_UNINT);
1201 NETEM_MTX_UNLOCK(ne);
1202 thread_block_parameter(netem_output_thread_cont, ne);
1203 /* NOTREACHED */
1204 __builtin_unreachable();
1205 }
1206
1207 static struct netem *
netem_create(const char * name,void * output_handle,int (* output)(void * handle,pktsched_pkt_t * pkts,uint32_t n_pkts),uint32_t output_max_batch_size)1208 netem_create(const char *name, void *output_handle,
1209 int (*output)(void *handle, pktsched_pkt_t *pkts, uint32_t n_pkts),
1210 uint32_t output_max_batch_size)
1211 {
1212 struct netem *ne;
1213
1214 ne = kalloc_type(struct netem, Z_WAITOK | Z_ZERO);
1215
1216 lck_mtx_init(&ne->netem_lock, &netem_lock_group, LCK_ATTR_NULL);
1217
1218 ne->netem_heap = heap_create(NETEM_HEAP_SIZE);
1219 ne->netem_flags = NETEMF_INITIALIZED;
1220 ne->netem_output_handle = output_handle;
1221 ne->netem_output = output;
1222 ne->netem_output_max_batch_size =
1223 MIN(output_max_batch_size, NETEM_MAX_BATCH_SIZE);
1224 ne->netem_output_thread = THREAD_NULL;
1225 if (output != NULL) {
1226 strlcpy(ne->netem_name, name, sizeof(ne->netem_name));
1227 if (kernel_thread_start(netem_output_thread_func, ne,
1228 &ne->netem_output_thread) != KERN_SUCCESS) {
1229 panic_plain("%s can't create thread", ne->netem_name);
1230 }
1231 }
1232
1233 return ne;
1234 }
1235
1236 void
netem_destroy(struct netem * ne)1237 netem_destroy(struct netem *ne)
1238 {
1239 uint64_t f = (1 * NSEC_PER_MSEC); /* 1 ms */
1240 uint64_t s = (1000 * NSEC_PER_MSEC); /* 1 sec */
1241 uint32_t i = 0;
1242 int ret = 0;
1243 uint64_t key = 0;
1244 pktsched_pkt_t pkt;
1245
1246 ASSERT(ne != NULL);
1247
1248 if (ne->netem_output_thread != THREAD_NULL) {
1249 ASSERT(ne->netem_flags & NETEMF_INITIALIZED);
1250 /* signal thread to begin self-termination */
1251 NETEM_MTX_LOCK(ne);
1252 ne->netem_flags |= NETEMF_TERMINATING;
1253
1254 /* and wait for thread to terminate */
1255 while (!(ne->netem_flags & NETEMF_TERMINATED)) {
1256 uint64_t t = 0;
1257 nanoseconds_to_absolutetime((i++ == 0) ? f : s, &t);
1258 clock_absolutetime_interval_to_deadline(t, &t);
1259 ASSERT(t != 0);
1260
1261 ne->netem_flags |= NETEMF_TERMINATEBLOCK;
1262 if (!(ne->netem_flags & NETEMF_RUNNING)) {
1263 thread_wakeup((caddr_t)&ne->netem_flags);
1264 }
1265 (void) assert_wait_deadline(&ne->netem_output_thread,
1266 THREAD_UNINT, t);
1267 NETEM_MTX_UNLOCK(ne);
1268 (void) thread_block(THREAD_CONTINUE_NULL);
1269 NETEM_MTX_LOCK(ne);
1270 ne->netem_flags &= ~NETEMF_TERMINATEBLOCK;
1271 }
1272 ASSERT(ne->netem_flags & NETEMF_TERMINATED);
1273 NETEM_MTX_UNLOCK(ne);
1274 ne->netem_output_thread = THREAD_NULL;
1275 }
1276 ASSERT(ne->netem_output_thread == THREAD_NULL);
1277
1278 lck_mtx_destroy(&ne->netem_lock, &netem_lock_group);
1279
1280 while ((ret = heap_extract(ne->netem_heap, &key, &pkt)) == 0) {
1281 pktsched_free_pkt(&pkt);
1282 }
1283 heap_destroy(ne->netem_heap);
1284
1285 kfree_type(struct netem, ne);
1286 }
1287
1288 static int
netem_check_params(const struct if_netem_params * p)1289 netem_check_params(const struct if_netem_params *p)
1290 {
1291 if (p->ifnetem_corruption_p > NETEM_PSCALE) {
1292 netem_log(NETEM_LOG_ERROR, "error: corruption_p %d > %d",
1293 p->ifnetem_corruption_p, NETEM_PSCALE);
1294 return EINVAL;
1295 }
1296
1297 if (p->ifnetem_duplication_p > NETEM_PSCALE) {
1298 netem_log(NETEM_LOG_ERROR, "error: duplication_p %d > %d",
1299 p->ifnetem_duplication_p, NETEM_PSCALE);
1300 return EINVAL;
1301 }
1302
1303 if (p->ifnetem_duplication_p > 0 &&
1304 p->ifnetem_latency_ms == 0) {
1305 /* we need to insert dup'ed packet with latency */
1306 netem_log(NETEM_LOG_ERROR,
1307 "error: duplication needs latency param");
1308 return EINVAL;
1309 }
1310
1311 if (p->ifnetem_latency_ms > 1000) {
1312 netem_log(NETEM_LOG_ERROR,
1313 "error: latency %d too big (> 1 sec)",
1314 p->ifnetem_latency_ms);
1315 return EINVAL;
1316 }
1317
1318 if (p->ifnetem_jitter_ms * 3 > p->ifnetem_latency_ms) {
1319 netem_log(NETEM_LOG_ERROR,
1320 "error: jitter %dms too big (latency %dms)",
1321 p->ifnetem_jitter_ms, p->ifnetem_latency_ms);
1322 return EINVAL;
1323 }
1324
1325 /* if gr_gl == 0 (no loss), other prob should all be zero */
1326 if (p->ifnetem_loss_p_gr_gl == 0 &&
1327 (p->ifnetem_loss_p_gr_bl != 0 ||
1328 p->ifnetem_loss_p_bl_br != 0 ||
1329 p->ifnetem_loss_p_bl_gr != 0 ||
1330 p->ifnetem_loss_p_br_bl != 0)) {
1331 netem_log(NETEM_LOG_ERROR,
1332 "error: loss params not all zero when p_gr_gl is zero");
1333 return EINVAL;
1334 }
1335
1336 /* check state machine transition prob integrity */
1337 if (p->ifnetem_loss_p_gr_gl > NETEM_PSCALE ||
1338 /* gr_gl = NETEM_PSCALE for total loss */
1339 p->ifnetem_loss_p_gr_bl > NETEM_PSCALE ||
1340 p->ifnetem_loss_p_bl_br > NETEM_PSCALE ||
1341 p->ifnetem_loss_p_bl_gr > NETEM_PSCALE ||
1342 p->ifnetem_loss_p_br_bl > NETEM_PSCALE ||
1343 p->ifnetem_loss_p_gr_gl + p->ifnetem_loss_p_gr_bl > NETEM_PSCALE ||
1344 p->ifnetem_loss_p_bl_br + p->ifnetem_loss_p_bl_gr > NETEM_PSCALE) {
1345 netem_log(NETEM_LOG_ERROR, "error: loss params too big");
1346 return EINVAL;
1347 }
1348
1349 if (p->ifnetem_reordering_p > NETEM_PSCALE) {
1350 netem_log(NETEM_LOG_ERROR, "error: reordering %d > %d",
1351 p->ifnetem_reordering_p, NETEM_PSCALE);
1352 return EINVAL;
1353 }
1354
1355 return 0;
1356 }
1357
1358 static void
netem_set_params(struct netem * ne,const struct if_netem_params * p)1359 netem_set_params(struct netem *ne, const struct if_netem_params *p)
1360 {
1361 NETEM_MTX_LOCK(ne);
1362
1363 struct token_bucket *tb = &ne->netem_bandwidth_model;
1364 if (p->ifnetem_bandwidth_bps == 0) {
1365 tb->depth = 0;
1366 tb->rate = 0;
1367 tb->token = 0;
1368 tb->last = 0;
1369 } else {
1370 tb->depth = p->ifnetem_bandwidth_bps;
1371 tb->rate = p->ifnetem_bandwidth_bps;
1372 tb->token = p->ifnetem_bandwidth_bps / 2;
1373 tb->last = mach_absolute_time();
1374 }
1375
1376 struct corruption *corr = &ne->netem_corruption_model;
1377 corr->corruption_p = p->ifnetem_corruption_p;
1378
1379 struct duplication *dup = &ne->netem_duplication_model;
1380 dup->duplication_p = p->ifnetem_duplication_p;
1381
1382 struct latency *late = &ne->netem_latency_model;
1383 late->latency_ms = p->ifnetem_latency_ms;
1384 late->jitter_ms = p->ifnetem_jitter_ms;
1385
1386 struct loss *loss = &ne->netem_loss_model;
1387 loss->state = GAP_RX;
1388 loss->p_gr_gl = p->ifnetem_loss_p_gr_gl;
1389 loss->p_gr_bl = p->ifnetem_loss_p_gr_bl;
1390 loss->p_bl_gr = p->ifnetem_loss_p_bl_gr;
1391 loss->p_bl_br = p->ifnetem_loss_p_bl_br;
1392 loss->p_br_bl = p->ifnetem_loss_p_br_bl;
1393
1394 struct reordering *r = &ne->netem_reordering_model;
1395 r->reordering_p = p->ifnetem_reordering_p;
1396
1397 netem_log(NETEM_LOG_INFO, "success: bandwidth %llu bps", tb->rate);
1398 netem_log(NETEM_LOG_INFO, "success: corruption %d%% ",
1399 corr->corruption_p);
1400 netem_log(NETEM_LOG_INFO, "success: duplication %d%%",
1401 dup->duplication_p);
1402 netem_log(NETEM_LOG_INFO, "success: latency_ms %d jitter_ms %d",
1403 late->latency_ms, late->jitter_ms);
1404 netem_log(NETEM_LOG_INFO, "changed loss p_gr_gl %d p_gr_bl %d "
1405 "p_bl_gr %d p_bl_br %d p_br_bl %d", loss->p_gr_gl, loss->p_gr_bl,
1406 loss->p_bl_gr, loss->p_bl_br, loss->p_br_bl);
1407 netem_log(NETEM_LOG_DEBUG, "success: reordering %d%%",
1408 r->reordering_p);
1409
1410 NETEM_MTX_UNLOCK(ne);
1411 }
1412
1413 void
netem_get_params(struct netem * ne,struct if_netem_params * p)1414 netem_get_params(struct netem *ne, struct if_netem_params *p)
1415 {
1416 ASSERT(ne != NULL);
1417 NETEM_MTX_LOCK(ne);
1418
1419 struct token_bucket *tb = &ne->netem_bandwidth_model;
1420 p->ifnetem_bandwidth_bps = tb->depth;
1421
1422 struct corruption *corr = &ne->netem_corruption_model;
1423 p->ifnetem_corruption_p = corr->corruption_p;
1424
1425 struct duplication *dup = &ne->netem_duplication_model;
1426 p->ifnetem_duplication_p = dup->duplication_p;
1427
1428 struct latency *late = &ne->netem_latency_model;
1429 p->ifnetem_latency_ms = late->latency_ms;
1430 p->ifnetem_jitter_ms = late->jitter_ms;
1431
1432 struct loss *loss = &ne->netem_loss_model;
1433 p->ifnetem_loss_p_gr_gl = loss->p_gr_gl;
1434 p->ifnetem_loss_p_gr_bl = loss->p_gr_bl;
1435 p->ifnetem_loss_p_bl_gr = loss->p_bl_gr;
1436 p->ifnetem_loss_p_bl_br = loss->p_bl_br;
1437 p->ifnetem_loss_p_br_bl = loss->p_br_bl;
1438
1439 struct reordering *r = &ne->netem_reordering_model;
1440 p->ifnetem_reordering_p = r->reordering_p;
1441
1442 NETEM_MTX_UNLOCK(ne);
1443 }
1444
1445 int
netem_config(struct netem ** ne,const char * name,const struct if_netem_params * p,void * output_handle,int (* output_func)(void * handle,pktsched_pkt_t * pkts,uint32_t n_pkts),uint32_t output_max_batch_size)1446 netem_config(struct netem **ne, const char *name,
1447 const struct if_netem_params *p, void *output_handle,
1448 int (*output_func)(void *handle, pktsched_pkt_t *pkts, uint32_t n_pkts),
1449 uint32_t output_max_batch_size)
1450 {
1451 struct netem *netem = NULL;
1452 boolean_t enable = TRUE;
1453 int ret = 0;
1454
1455 if (p == NULL || (
1456 p->ifnetem_bandwidth_bps == 0 &&
1457 p->ifnetem_corruption_p == 0 &&
1458 p->ifnetem_duplication_p == 0 &&
1459 p->ifnetem_latency_ms == 0 &&
1460 p->ifnetem_jitter_ms == 0 &&
1461 p->ifnetem_loss_p_gr_gl == 0 &&
1462 p->ifnetem_loss_p_gr_bl == 0 &&
1463 p->ifnetem_loss_p_bl_br == 0 &&
1464 p->ifnetem_loss_p_bl_gr == 0 &&
1465 p->ifnetem_loss_p_br_bl == 0 &&
1466 p->ifnetem_reordering_p == 0)) {
1467 enable = FALSE;
1468 }
1469
1470 ret = netem_check_params(p);
1471 if (ret != 0) {
1472 goto done;
1473 }
1474
1475 if (enable) {
1476 if (*ne == NULL) {
1477 netem_log(NETEM_LOG_INFO, "netem create %s", name);
1478 netem = netem_create(name, output_handle, output_func,
1479 output_max_batch_size);
1480 if (netem == NULL) {
1481 return ENOMEM;
1482 }
1483 atomic_set_ptr(ne, netem);
1484 }
1485 netem_set_params(*ne, p);
1486 } else {
1487 netem_log(NETEM_LOG_INFO, "netem disable %s", name);
1488 if (*ne != NULL) {
1489 netem = *ne;
1490 atomic_set_ptr(ne, NULL);
1491 netem_log(NETEM_LOG_INFO, "netem destroy %s", name);
1492 netem_destroy(netem);
1493 }
1494 ret = 0;
1495 }
1496
1497 done:
1498 netem_log(NETEM_LOG_INFO, "netem config ret %d", ret);
1499 return ret;
1500 }
1501
1502 #else /* !CONFIG_NETEM */
1503
1504 int
netem_config(struct netem ** ne,const char * name,const struct if_netem_params * p,void * output_handle,int (* output_func)(void * handle,pktsched_pkt_t * pkts,uint32_t n_pkts),uint32_t output_max_batch_size)1505 netem_config(struct netem **ne, const char *name,
1506 const struct if_netem_params *p, void *output_handle,
1507 int (*output_func)(void *handle, pktsched_pkt_t *pkts, uint32_t n_pkts),
1508 uint32_t output_max_batch_size)
1509 {
1510 #pragma unused(ne, name, p, output_handle, output_func, output_max_batch_size)
1511 printf("%s error %d: unavailable on this platform\n", __func__, ENOTSUP);
1512 return ENOTSUP;
1513 }
1514
1515 void
1516 __attribute__((noreturn))
netem_get_params(struct netem * ne,struct if_netem_params * p)1517 netem_get_params(struct netem *ne, struct if_netem_params *p)
1518 {
1519 #pragma unused(ne, p)
1520 panic("unexpected netem call");
1521 }
1522
1523 void
1524 __attribute__((noreturn))
netem_destroy(struct netem * ne)1525 netem_destroy(struct netem *ne)
1526 {
1527 #pragma unused(ne)
1528 panic("unexpected netem call");
1529 }
1530
1531 int
netem_enqueue(struct netem * ne,classq_pkt_t * p,boolean_t * pdrop)1532 netem_enqueue(struct netem *ne, classq_pkt_t *p, boolean_t *pdrop)
1533 {
1534 #pragma unused(ne, p, pdrop)
1535 panic("unexpected netem call");
1536 return 0;
1537 }
1538 #endif /* !CONFIG_NETEM */
1539