1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <pexpert/pexpert.h> /* for PE_parse_boot_argn */
31 #include <sys/codesign.h> /* for csproc_get_platform_binary */
32 #include <sys/signalvar.h> /* for psignal_with_reason */
33 #include <sys/reason.h>
34 #include <sys/kern_memorystatus.h>
35 #if CONFIG_MACF
36 #include <security/mac_framework.h>
37 #endif /* CONFIG_MACF */
38
39 #ifndef htole16
40 #if BYTE_ORDER == LITTLE_ENDIAN
41 #define htole16(x) ((uint16_t)(x))
42 #else /* BYTE_ORDER != LITTLE_ENDIAN */
43 #define htole16(x) bswap16((x))
44 #endif /* BYTE_ORDER == LITTLE_ENDIAN */
45 #endif /* htole16 */
46
47 LCK_GRP_DECLARE(sk_lock_group, "sk_lock");
48 LCK_ATTR_DECLARE(sk_lock_attr, 0, 0);
49 LCK_MTX_DECLARE_ATTR(sk_lock, &sk_lock_group, &sk_lock_attr);
50
51 static void skywalk_fini(void);
52 static int sk_priv_chk(proc_t, kauth_cred_t, int);
53
54 static int __sk_inited = 0;
55 #if (DEVELOPMENT || DEBUG)
56 size_t sk_copy_thres = SK_COPY_THRES;
57 uint64_t sk_verbose;
58 #endif /* DEVELOPMENT || DEBUG */
59 uint32_t sk_debug;
60 uint64_t sk_features =
61 #if SKYWALK
62 SK_FEATURE_SKYWALK |
63 #endif
64 #if DEVELOPMENT
65 SK_FEATURE_DEVELOPMENT |
66 #endif
67 #if DEBUG
68 SK_FEATURE_DEBUG |
69 #endif
70 #if CONFIG_NEXUS_FLOWSWITCH
71 SK_FEATURE_NEXUS_FLOWSWITCH |
72 #endif
73 #if CONFIG_NEXUS_MONITOR
74 SK_FEATURE_NEXUS_MONITOR |
75 #endif
76 #if CONFIG_NEXUS_NETIF
77 SK_FEATURE_NEXUS_NETIF |
78 #endif
79 #if CONFIG_NEXUS_USER_PIPE
80 SK_FEATURE_NEXUS_USER_PIPE |
81 #endif
82 #if CONFIG_NEXUS_KERNEL_PIPE
83 SK_FEATURE_NEXUS_KERNEL_PIPE |
84 #endif
85 #if CONFIG_NEXUS_KERNEL_PIPE && (DEVELOPMENT || DEBUG)
86 SK_FEATURE_NEXUS_KERNEL_PIPE_LOOPBACK |
87 #endif
88 #if (DEVELOPMENT || DEBUG)
89 SK_FEATURE_DEV_OR_DEBUG |
90 #endif
91 0;
92
93 uint32_t sk_opp_defunct = 0; /* opportunistic defunct */
94
95 /* checksum offload is generic to any nexus (not specific to flowswitch) */
96 uint32_t sk_cksum_tx = 1; /* advertise outbound offload */
97 uint32_t sk_cksum_rx = 1; /* perform inbound checksum offload */
98
99 /* guard pages */
100 uint32_t sk_guard = 0; /* guard pages (0: disable) */
101 #define SK_GUARD_MIN 1 /* minimum # of guard pages */
102 #define SK_GUARD_MAX 4 /* maximum # of guard pages */
103 uint32_t sk_headguard_sz = SK_GUARD_MIN; /* # of leading guard pages */
104 uint32_t sk_tailguard_sz = SK_GUARD_MIN; /* # of trailing guard pages */
105
106 #if (DEVELOPMENT || DEBUG)
107 uint32_t sk_txring_sz = 0; /* flowswitch */
108 uint32_t sk_rxring_sz = 0; /* flowswitch */
109 uint32_t sk_net_txring_sz = 0; /* netif adapter */
110 uint32_t sk_net_rxring_sz = 0; /* netif adapter */
111 uint32_t sk_min_pool_size = 0; /* minimum packet pool size */
112 #endif /* !DEVELOPMENT && !DEBUG */
113
114 uint32_t sk_max_flows = NX_FLOWADV_DEFAULT;
115 uint32_t sk_fadv_nchunks; /* # of FO_FLOWADV_CHUNK in bitmap */
116 uint32_t sk_netif_compat_txmodel = NETIF_COMPAT_TXMODEL_DEFAULT;
117 uint32_t sk_netif_native_txmodel = NETIF_NATIVE_TXMODEL_DEFAULT;
118 /*
119 * Configures the RX aggregation logic for TCP in flowswitch.
120 * A non-zero value enables the aggregation logic, with the maximum
121 * aggregation length (in bytes) limited to this value.
122 */
123 uint32_t sk_fsw_rx_agg_tcp = 16384;
124
125 /*
126 * Forces the RX host path to use or not use aggregation, regardless of the
127 * existence of filters (see sk_fsw_rx_agg_tcp_host_t for valid values).
128 */
129 uint32_t sk_fsw_rx_agg_tcp_host = SK_FSW_RX_AGG_TCP_HOST_AUTO;
130
131 /*
132 * Configures the skywalk infrastructure for handling TCP TX aggregation.
133 * A non-zero value enables the support.
134 */
135 uint32_t sk_fsw_tx_agg_tcp = 1;
136 /*
137 * Configuration to limit the number of buffers for flowswitch VP channel.
138 */
139 uint32_t sk_fsw_max_bufs = 0;
140
141 /* list of interfaces that allow direct open from userspace */
142 #define SK_NETIF_DIRECT_MAX 8
143 char sk_netif_direct[SK_NETIF_DIRECT_MAX][IFXNAMSIZ];
144 uint32_t sk_netif_direct_cnt = 0;
145
146 uint16_t sk_tx_delay_qlen = 16; /* packets */
147 uint16_t sk_tx_delay_timeout = (1 * 1000); /* microseconds */
148
149 #define SK_NETIF_COMPAT_AUX_CELL_TX_RING_SZ 64
150 #define SK_NETIF_COMPAT_AUX_CELL_RX_RING_SZ 64
151 uint32_t sk_netif_compat_aux_cell_tx_ring_sz =
152 SK_NETIF_COMPAT_AUX_CELL_TX_RING_SZ;
153 uint32_t sk_netif_compat_aux_cell_rx_ring_sz =
154 SK_NETIF_COMPAT_AUX_CELL_RX_RING_SZ;
155
156 /* Wi-Fi Access Point */
157 #define SK_NETIF_COMPAT_WAP_TX_RING_SZ 128
158 #define SK_NETIF_COMPAT_WAP_RX_RING_SZ 128
159 uint32_t sk_netif_compat_wap_tx_ring_sz = SK_NETIF_COMPAT_WAP_TX_RING_SZ;
160 uint32_t sk_netif_compat_wap_rx_ring_sz = SK_NETIF_COMPAT_WAP_RX_RING_SZ;
161
162 /* AWDL */
163 #define SK_NETIF_COMPAT_AWDL_TX_RING_SZ 128
164 #define SK_NETIF_COMPAT_AWDL_RX_RING_SZ 128
165 uint32_t sk_netif_compat_awdl_tx_ring_sz = SK_NETIF_COMPAT_AWDL_TX_RING_SZ;
166 uint32_t sk_netif_compat_awdl_rx_ring_sz = SK_NETIF_COMPAT_AWDL_RX_RING_SZ;
167
168 /* Wi-Fi Infrastructure */
169 #define SK_NETIF_COMPAT_WIF_TX_RING_SZ 128
170 #define SK_NETIF_COMPAT_WIF_RX_RING_SZ 128
171 uint32_t sk_netif_compat_wif_tx_ring_sz = SK_NETIF_COMPAT_WIF_TX_RING_SZ;
172 uint32_t sk_netif_compat_wif_rx_ring_sz = SK_NETIF_COMPAT_WIF_RX_RING_SZ;
173
174 #define SK_NETIF_COMPAT_USB_ETH_TX_RING_SZ 128
175 #define SK_NETIF_COMPAT_USB_ETH_RX_RING_SZ 128
176 uint32_t sk_netif_compat_usb_eth_tx_ring_sz =
177 SK_NETIF_COMPAT_USB_ETH_TX_RING_SZ;
178 uint32_t sk_netif_compat_usb_eth_rx_ring_sz =
179 SK_NETIF_COMPAT_USB_ETH_RX_RING_SZ;
180
181 #define SK_NETIF_COMPAT_RX_MBQ_LIMIT 8192
182 int sk_netif_compat_rx_mbq_limit = SK_NETIF_COMPAT_RX_MBQ_LIMIT;
183
184 uint32_t sk_netif_tx_mit = SK_NETIF_MIT_AUTO;
185 uint32_t sk_netif_rx_mit = SK_NETIF_MIT_AUTO;
186 char sk_ll_prefix[IFNAMSIZ] = "llw";
187 uint32_t sk_rx_sync_packets = 1;
188 uint32_t sk_channel_buflet_alloc = 0;
189
190 SYSCTL_NODE(_kern, OID_AUTO, skywalk, CTLFLAG_RW | CTLFLAG_LOCKED,
191 0, "Skywalk parameters");
192 SYSCTL_NODE(_kern_skywalk, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_LOCKED,
193 0, "Skywalk statistics");
194
195 SYSCTL_OPAQUE(_kern_skywalk, OID_AUTO, features, CTLFLAG_RD | CTLFLAG_LOCKED,
196 &sk_features, sizeof(sk_features), "-", "Skywalk features");
197
198 #if (DEVELOPMENT || DEBUG)
199 SYSCTL_QUAD(_kern_skywalk, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
200 &sk_verbose, "Skywalk verbose mode");
201 SYSCTL_UINT(_kern_skywalk, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
202 &sk_debug, 0, "Skywalk debug mode");
203 SYSCTL_LONG(_kern_skywalk, OID_AUTO, sk_copy_thres, CTLFLAG_RW | CTLFLAG_LOCKED,
204 &sk_copy_thres, "Skywalk copy threshold");
205 static int __priv_check = 1;
206 SYSCTL_INT(_kern_skywalk, OID_AUTO, priv_check, CTLFLAG_RW | CTLFLAG_LOCKED,
207 &__priv_check, 0, "Skywalk privilege check");
208 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_opp_defunct, CTLFLAG_RW | CTLFLAG_LOCKED,
209 &sk_opp_defunct, 0, "Defunct opportunistically");
210 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_cksum_tx, CTLFLAG_RW | CTLFLAG_LOCKED,
211 &sk_cksum_tx, 0, "Advertise (and perform) outbound checksum offload");
212 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_cksum_rx, CTLFLAG_RW | CTLFLAG_LOCKED,
213 &sk_cksum_rx, 0, "Perform inbound checksum offload");
214 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_rx_sync_packets, CTLFLAG_RW | CTLFLAG_LOCKED,
215 &sk_rx_sync_packets, 0, "Enable RX sync packets");
216 SYSCTL_UINT(_kern_skywalk, OID_AUTO, chan_buf_alloc,
217 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_channel_buflet_alloc, 0,
218 "channel buflet allocation (enable/disable)");
219 #endif /* !DEVELOPMENT && !DEBUG */
220
221 #if (DEVELOPMENT || DEBUG)
222 uint32_t sk_inject_error_rmask = 0x3;
223 SYSCTL_UINT(_kern_skywalk, OID_AUTO, inject_error_rmask,
224 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_inject_error_rmask, 0x3, "");
225 #endif /* !DEVELOPMENT && !DEBUG */
226
227 #if (DEVELOPMENT || DEBUG)
228 static void skywalk_self_tests(void);
229 #endif /* (DEVELOPMENT || DEBUG) */
230
231 #define SKMEM_TAG_SYSCTL_BUF "com.apple.skywalk.sysctl_buf"
232 SKMEM_TAG_DEFINE(skmem_tag_sysctl_buf, SKMEM_TAG_SYSCTL_BUF);
233
234 #define SKMEM_TAG_OID "com.apple.skywalk.skoid"
235 SKMEM_TAG_DEFINE(skmem_tag_oid, SKMEM_TAG_OID);
236
237 #if (SK_LOG || DEVELOPMENT || DEBUG)
238 #define SKMEM_TAG_DUMP "com.apple.skywalk.dump"
239 static SKMEM_TAG_DEFINE(skmem_tag_dump, SKMEM_TAG_DUMP);
240
241 static char *sk_dump_buf;
242 #define SK_DUMP_BUF_SIZE 2048
243 #define SK_DUMP_BUF_ALIGN 16
244 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
245
246 __startup_func
247 void
__sk_tag_make(const struct sk_tag_spec * spec)248 __sk_tag_make(const struct sk_tag_spec *spec)
249 {
250 *spec->skt_var = kern_allocation_name_allocate(spec->skt_name, 0);
251 }
252
253 boolean_t
skywalk_netif_direct_enabled(void)254 skywalk_netif_direct_enabled(void)
255 {
256 return sk_netif_direct_cnt > 0;
257 }
258
259 boolean_t
skywalk_netif_direct_allowed(const char * ifname)260 skywalk_netif_direct_allowed(const char *ifname)
261 {
262 uint32_t i;
263
264 for (i = 0; i < sk_netif_direct_cnt; i++) {
265 if (strncmp(ifname, sk_netif_direct[i], IFXNAMSIZ) == 0) {
266 return TRUE;
267 }
268 }
269 return FALSE;
270 }
271
272 #if (DEVELOPMENT || DEBUG)
273 static void
parse_netif_direct(void)274 parse_netif_direct(void)
275 {
276 char buf[(IFXNAMSIZ + 1) * SK_NETIF_DIRECT_MAX];
277 size_t i, curr, len, iflen;
278
279 if (!PE_parse_boot_arg_str("sk_netif_direct", buf, sizeof(buf))) {
280 return;
281 }
282
283 curr = 0;
284 len = strlen(buf);
285 for (i = 0; i < len + 1 &&
286 sk_netif_direct_cnt < SK_NETIF_DIRECT_MAX; i++) {
287 if (buf[i] != ',' && buf[i] != '\0') {
288 continue;
289 }
290
291 buf[i] = '\0';
292 iflen = i - curr;
293 if (iflen > 0 && iflen < IFXNAMSIZ) {
294 (void) strlcpy(sk_netif_direct[sk_netif_direct_cnt],
295 buf + curr, IFXNAMSIZ);
296 sk_netif_direct_cnt++;
297 }
298 curr = i + 1;
299 }
300 }
301 #endif /* DEVELOPMENT || DEBUG */
302
303 static void
skywalk_fini(void)304 skywalk_fini(void)
305 {
306 SK_LOCK_ASSERT_HELD();
307
308 if (__sk_inited) {
309 #if (DEVELOPMENT || DEBUG)
310 skmem_test_fini();
311 cht_test_fini();
312 #endif /* (DEVELOPMENT || DEBUG) */
313 channel_fini();
314 nexus_fini();
315 skmem_fini();
316
317 #if (SK_LOG || DEVELOPMENT || DEBUG)
318 if (sk_dump_buf != NULL) {
319 sk_free_data(sk_dump_buf, SK_DUMP_BUF_SIZE);
320 sk_dump_buf = NULL;
321 }
322 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
323
324 __sk_inited = 0;
325 }
326 }
327
328 int
skywalk_init(void)329 skywalk_init(void)
330 {
331 int error;
332
333 VERIFY(!__sk_inited);
334
335 _CASSERT(sizeof(kern_packet_t) == sizeof(uint64_t));
336 _CASSERT(sizeof(bitmap_t) == sizeof(uint64_t));
337
338 #if (DEVELOPMENT || DEBUG)
339 PE_parse_boot_argn("sk_debug", &sk_debug, sizeof(sk_debug));
340 PE_parse_boot_argn("sk_verbose", &sk_verbose, sizeof(sk_verbose));
341 (void) PE_parse_boot_argn("sk_opp_defunct", &sk_opp_defunct,
342 sizeof(sk_opp_defunct));
343 (void) PE_parse_boot_argn("sk_cksum_tx", &sk_cksum_tx,
344 sizeof(sk_cksum_tx));
345 (void) PE_parse_boot_argn("sk_cksum_rx", &sk_cksum_rx,
346 sizeof(sk_cksum_rx));
347 (void) PE_parse_boot_argn("sk_txring_sz", &sk_txring_sz,
348 sizeof(sk_txring_sz));
349 (void) PE_parse_boot_argn("sk_rxring_sz", &sk_rxring_sz,
350 sizeof(sk_rxring_sz));
351 (void) PE_parse_boot_argn("sk_net_txring_sz", &sk_net_txring_sz,
352 sizeof(sk_net_txring_sz));
353 (void) PE_parse_boot_argn("sk_net_rxring_sz", &sk_net_rxring_sz,
354 sizeof(sk_net_rxring_sz));
355 (void) PE_parse_boot_argn("sk_max_flows", &sk_max_flows,
356 sizeof(sk_max_flows));
357 (void) PE_parse_boot_argn("sk_native_txmodel", &sk_netif_native_txmodel,
358 sizeof(sk_netif_native_txmodel));
359 (void) PE_parse_boot_argn("sk_compat_txmodel", &sk_netif_compat_txmodel,
360 sizeof(sk_netif_compat_txmodel));
361 (void) PE_parse_boot_argn("sk_tx_delay_qlen", &sk_tx_delay_qlen,
362 sizeof(sk_tx_delay_qlen));
363 (void) PE_parse_boot_argn("sk_ts_delay_timeout", &sk_tx_delay_timeout,
364 sizeof(sk_tx_delay_timeout));
365 (void) PE_parse_boot_argn("sk_compat_aux_cell_tx_ring_sz",
366 &sk_netif_compat_aux_cell_tx_ring_sz,
367 sizeof(sk_netif_compat_aux_cell_tx_ring_sz));
368 (void) PE_parse_boot_argn("sk_compat_aux_cell_rx_ring_sz",
369 &sk_netif_compat_aux_cell_rx_ring_sz,
370 sizeof(sk_netif_compat_aux_cell_rx_ring_sz));
371 (void) PE_parse_boot_argn("sk_compat_wap_tx_ring_sz",
372 &sk_netif_compat_wap_tx_ring_sz,
373 sizeof(sk_netif_compat_wap_tx_ring_sz));
374 (void) PE_parse_boot_argn("sk_compat_wap_rx_ring_sz",
375 &sk_netif_compat_wap_rx_ring_sz,
376 sizeof(sk_netif_compat_wap_rx_ring_sz));
377 (void) PE_parse_boot_argn("sk_compat_awdl_tx_ring_sz",
378 &sk_netif_compat_awdl_tx_ring_sz,
379 sizeof(sk_netif_compat_awdl_tx_ring_sz));
380 (void) PE_parse_boot_argn("sk_compat_awdl_rx_ring_sz",
381 &sk_netif_compat_awdl_rx_ring_sz,
382 sizeof(sk_netif_compat_awdl_rx_ring_sz));
383 (void) PE_parse_boot_argn("sk_compat_wif_tx_ring_sz",
384 &sk_netif_compat_wif_tx_ring_sz,
385 sizeof(sk_netif_compat_wif_tx_ring_sz));
386 (void) PE_parse_boot_argn("sk_compat_wif_rx_ring_sz",
387 &sk_netif_compat_wif_rx_ring_sz,
388 sizeof(sk_netif_compat_wif_rx_ring_sz));
389 (void) PE_parse_boot_argn("sk_compat_usb_eth_tx_ring_sz",
390 &sk_netif_compat_usb_eth_tx_ring_sz,
391 sizeof(sk_netif_compat_usb_eth_tx_ring_sz));
392 (void) PE_parse_boot_argn("sk_compat_usb_eth_rx_ring_sz",
393 &sk_netif_compat_usb_eth_rx_ring_sz,
394 sizeof(sk_netif_compat_usb_eth_rx_ring_sz));
395 (void) PE_parse_boot_argn("sk_compat_rx_mbq_limit",
396 &sk_netif_compat_rx_mbq_limit, sizeof(sk_netif_compat_rx_mbq_limit));
397 (void) PE_parse_boot_argn("sk_netif_tx_mit",
398 &sk_netif_tx_mit, sizeof(sk_netif_tx_mit));
399 (void) PE_parse_boot_argn("sk_netif_rx_mit",
400 &sk_netif_rx_mit, sizeof(sk_netif_rx_mit));
401 (void) PE_parse_boot_arg_str("sk_ll_prefix", sk_ll_prefix,
402 sizeof(sk_ll_prefix));
403 parse_netif_direct();
404 (void) PE_parse_boot_argn("sk_fsw_rx_agg_tcp", &sk_fsw_rx_agg_tcp,
405 sizeof(sk_fsw_rx_agg_tcp));
406 (void) PE_parse_boot_argn("sk_fsw_tx_agg_tcp", &sk_fsw_tx_agg_tcp,
407 sizeof(sk_fsw_tx_agg_tcp));
408 (void) PE_parse_boot_argn("sk_fsw_max_bufs", &sk_fsw_max_bufs,
409 sizeof(sk_fsw_max_bufs));
410 (void) PE_parse_boot_argn("sk_rx_sync_packets", &sk_rx_sync_packets,
411 sizeof(sk_rx_sync_packets));
412 (void) PE_parse_boot_argn("sk_chan_buf_alloc", &sk_channel_buflet_alloc,
413 sizeof(sk_channel_buflet_alloc));
414 (void) PE_parse_boot_argn("sk_guard", &sk_guard, sizeof(sk_guard));
415 (void) PE_parse_boot_argn("sk_headguard_sz", &sk_headguard_sz,
416 sizeof(sk_headguard_sz));
417 (void) PE_parse_boot_argn("sk_tailguard_sz", &sk_tailguard_sz,
418 sizeof(sk_tailguard_sz));
419 (void) PE_parse_boot_argn("sk_min_pool_size", &sk_min_pool_size,
420 sizeof(sk_min_pool_size));
421 #endif /* DEVELOPMENT || DEBUG */
422
423 if (sk_max_flows == 0) {
424 sk_max_flows = NX_FLOWADV_DEFAULT;
425 } else if (sk_max_flows > NX_FLOWADV_MAX) {
426 sk_max_flows = NX_FLOWADV_MAX;
427 }
428
429 if (sk_netif_tx_mit > SK_NETIF_MIT_MAX) {
430 sk_netif_tx_mit = SK_NETIF_MIT_MAX;
431 }
432 if (sk_netif_rx_mit > SK_NETIF_MIT_MAX) {
433 sk_netif_rx_mit = SK_NETIF_MIT_MAX;
434 }
435
436 sk_fadv_nchunks = (uint32_t)P2ROUNDUP(sk_max_flows, FO_FLOWADV_CHUNK) /
437 FO_FLOWADV_CHUNK;
438
439 if (sk_guard) {
440 uint32_t sz;
441 /* leading guard page(s) */
442 if (sk_headguard_sz == 0) {
443 read_frandom(&sz, sizeof(sz));
444 sk_headguard_sz = (sz % (SK_GUARD_MAX + 1));
445 } else if (sk_headguard_sz > SK_GUARD_MAX) {
446 sk_headguard_sz = SK_GUARD_MAX;
447 }
448 if (sk_headguard_sz < SK_GUARD_MIN) {
449 sk_headguard_sz = SK_GUARD_MIN;
450 }
451 /* trailing guard page(s) */
452 if (sk_tailguard_sz == 0) {
453 read_frandom(&sz, sizeof(sz));
454 sk_tailguard_sz = (sz % (SK_GUARD_MAX + 1));
455 } else if (sk_tailguard_sz > SK_GUARD_MAX) {
456 sk_tailguard_sz = SK_GUARD_MAX;
457 }
458 if (sk_tailguard_sz < SK_GUARD_MIN) {
459 sk_tailguard_sz = SK_GUARD_MIN;
460 }
461 } else {
462 sk_headguard_sz = sk_tailguard_sz = SK_GUARD_MIN;
463 }
464 ASSERT(sk_headguard_sz >= SK_GUARD_MIN);
465 ASSERT(sk_headguard_sz <= SK_GUARD_MAX);
466 ASSERT(sk_tailguard_sz >= SK_GUARD_MIN);
467 ASSERT(sk_tailguard_sz <= SK_GUARD_MAX);
468
469 __sk_inited = 1;
470
471 SK_LOCK();
472 skmem_init();
473 error = nexus_init();
474 if (error == 0) {
475 error = channel_init();
476 }
477 if (error != 0) {
478 skywalk_fini();
479 }
480 SK_UNLOCK();
481
482 if (error == 0) {
483 #if (SK_LOG || DEVELOPMENT || DEBUG)
484 /* allocate space for sk_dump_buf */
485 sk_dump_buf = sk_alloc_data(SK_DUMP_BUF_SIZE, Z_WAITOK | Z_NOFAIL,
486 skmem_tag_dump);
487 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
488
489 netns_init();
490 protons_init();
491
492 #if (DEVELOPMENT || DEBUG)
493 skywalk_self_tests();
494 skmem_test_init();
495 cht_test_init();
496 #endif /* DEVELOPMENT || DEBUG */
497 }
498
499 return error;
500 }
501
502 /*
503 * csproc_get_platform_binary() returns non-zero if the process is platform
504 * code, which means that it is considered part of the Operating System.
505 * On iOS, that means it's contained in the trust cache or a loaded one.
506 * On macOS, everything signed by B&I is currently platform code, but the
507 * policy in general is subject to change. Thus this is an approximate.
508 */
509 boolean_t
skywalk_check_platform_binary(proc_t p)510 skywalk_check_platform_binary(proc_t p)
511 {
512 return (csproc_get_platform_binary(p) == 0) ? FALSE : TRUE;
513 }
514
515 static int
sk_priv_chk(proc_t p,kauth_cred_t cred,int priv)516 sk_priv_chk(proc_t p, kauth_cred_t cred, int priv)
517 {
518 #pragma unused(p)
519 int ret = priv_check_cred(cred, priv, 0);
520
521 #if (DEVELOPMENT || DEBUG)
522 if (ret != 0) {
523 const char *pstr;
524
525 switch (priv) {
526 case PRIV_SKYWALK_REGISTER_USER_PIPE:
527 pstr = "com.apple.private.skywalk.register-user-pipe";
528 break;
529
530 case PRIV_SKYWALK_REGISTER_KERNEL_PIPE:
531 pstr = "com.apple.private.skywalk.register-kernel-pipe";
532 break;
533
534 case PRIV_SKYWALK_REGISTER_NET_IF:
535 pstr = "com.apple.private.skywalk.register-net-if";
536 break;
537
538 case PRIV_SKYWALK_REGISTER_FLOW_SWITCH:
539 pstr = "com.apple.private.skywalk.register-flow-switch";
540 break;
541
542 case PRIV_SKYWALK_OBSERVE_ALL:
543 pstr = "com.apple.private.skywalk.observe-all";
544 break;
545
546 case PRIV_SKYWALK_OBSERVE_STATS:
547 pstr = "com.apple.private.skywalk.observe-stats";
548 break;
549
550 case PRIV_SKYWALK_LOW_LATENCY_CHANNEL:
551 pstr = "com.apple.private.skywalk.low-latency-channel";
552 break;
553
554 default:
555 pstr = "unknown";
556 break;
557 }
558
559 #if SK_LOG
560 if (__priv_check) {
561 SK_DF(SK_VERB_PRIV, "%s(%d) insufficient privilege %d "
562 "(\"%s\") err %d", sk_proc_name_address(p),
563 sk_proc_pid(p), priv, pstr, ret);
564 } else {
565 SK_DF(SK_VERB_PRIV, "%s(%d) IGNORING missing privilege "
566 "%d (\"%s\") err %d", sk_proc_name_address(p),
567 sk_proc_pid(p), priv, pstr, ret);
568 }
569 #endif /* SK_LOG */
570
571 /* ignore privilege check failures if requested */
572 if (!__priv_check) {
573 ret = 0;
574 }
575 }
576 #endif /* !DEVELOPMENT && !DEBUG */
577
578 return ret;
579 }
580
581 int
skywalk_priv_check_cred(proc_t p,kauth_cred_t cred,int priv)582 skywalk_priv_check_cred(proc_t p, kauth_cred_t cred, int priv)
583 {
584 return sk_priv_chk(p, cred, priv);
585 }
586
587 int
skywalk_priv_check_proc_cred(proc_t p,int priv)588 skywalk_priv_check_proc_cred(proc_t p, int priv)
589 {
590 int ret;
591 kauth_cred_t cred = kauth_cred_proc_ref(p);
592 ret = sk_priv_chk(p, cred, priv);
593 kauth_cred_unref(&cred);
594
595 return ret;
596 }
597
598 #if CONFIG_MACF
599 int
skywalk_mac_system_check_proc_cred(proc_t p,const char * info_type)600 skywalk_mac_system_check_proc_cred(proc_t p, const char *info_type)
601 {
602 int ret;
603 kauth_cred_t cred = kauth_cred_proc_ref(p);
604 ret = mac_system_check_info(cred, info_type);
605 kauth_cred_unref(&cred);
606
607 return ret;
608 }
609 #endif /* CONFIG_MACF */
610
611 /*
612 * Scan thru the list of privileges needed before we allow someone
613 * to open a handle to the Nexus controller. This should be done
614 * at nxctl_create() time, and additional privilege check specific
615 * to the operation (e.g. register, etc.) should be done afterwards.
616 */
617 int
skywalk_nxctl_check_privileges(proc_t p,kauth_cred_t cred)618 skywalk_nxctl_check_privileges(proc_t p, kauth_cred_t cred)
619 {
620 int ret = 0;
621
622 if (p == kernproc) {
623 goto done;
624 }
625
626 do {
627 /*
628 * Check for observe-{stats,all} entitlements first
629 * before the rest, to account for nexus controller
630 * clients that don't need anything but statistics;
631 * it would help quiesce sandbox violation warnings.
632 */
633 if ((ret = sk_priv_chk(p, cred,
634 PRIV_SKYWALK_OBSERVE_STATS)) == 0) {
635 break;
636 }
637 if ((ret = sk_priv_chk(p, cred,
638 PRIV_SKYWALK_OBSERVE_ALL)) == 0) {
639 break;
640 }
641 if ((ret = sk_priv_chk(p, cred,
642 PRIV_SKYWALK_REGISTER_USER_PIPE)) == 0) {
643 break;
644 }
645 if ((ret = sk_priv_chk(p, cred,
646 PRIV_SKYWALK_REGISTER_KERNEL_PIPE)) == 0) {
647 break;
648 }
649 if ((ret = sk_priv_chk(p, cred,
650 PRIV_SKYWALK_REGISTER_NET_IF)) == 0) {
651 break;
652 }
653 if ((ret = sk_priv_chk(p, cred,
654 PRIV_SKYWALK_REGISTER_FLOW_SWITCH)) == 0) {
655 break;
656 }
657 /* none set, so too bad */
658 ret = EPERM;
659 } while (0);
660
661 #if (DEVELOPMENT || DEBUG)
662 if (ret != 0) {
663 SK_ERR("%s(%d) insufficient privilege to open nexus controller "
664 "err %d", sk_proc_name_address(p), sk_proc_pid(p), ret);
665 }
666 #endif /* !DEVELOPMENT && !DEBUG */
667 done:
668 return ret;
669 }
670
671 void
sk_gen_guard_id(boolean_t isch,const uuid_t uuid,guardid_t * guard)672 sk_gen_guard_id(boolean_t isch, const uuid_t uuid, guardid_t *guard)
673 {
674 #define GUARD_CH_SIG 0x4348 /* 'CH' */
675 #define GUARD_NX_SIG 0x4e58 /* 'NX' */
676 union {
677 uint8_t _u8[8];
678 uint16_t _u16[4];
679 uint64_t _u64;
680 } __u;
681
682 read_random(&__u._u16[0], sizeof(uint16_t));
683 bcopy(uuid, (void *)&__u._u16[1], sizeof(uint16_t));
684 __u._u16[2] = htole16(isch ? GUARD_CH_SIG : GUARD_NX_SIG);
685 __u._u16[3] = htole16(0x534b); /* 'SK' */
686 VERIFY(__u._u64 != 0);
687
688 bzero(guard, sizeof(*guard));
689 bcopy((void *)&__u._u64, guard, MIN(sizeof(*guard),
690 sizeof(uint64_t)));
691 }
692
693
694 extern const char *
sk_uuid_unparse(const uuid_t uu,uuid_string_t out)695 sk_uuid_unparse(const uuid_t uu, uuid_string_t out)
696 {
697 uuid_unparse_upper(uu, out);
698 return (const char *)out;
699 }
700
701 #if SK_LOG
702 /*
703 * packet-dump function, user-supplied or static buffer.
704 * The destination buffer must be at least 30+4*len
705 *
706 * @param p
707 * buffer to be dumped.
708 * @param len
709 * buffer's total length.
710 * @param dumplen
711 * length to be dumped.
712 * @param dst
713 * destination char buffer. sk_dump_buf would be used if dst is NULL.
714 * @param lim
715 * destination char buffer max length. Not used if dst is NULL.
716 *
717 */
718 const char *
sk_dump(const char * label,const void * obj,int len,int dumplen,char * dst,int lim)719 sk_dump(const char *label, const void *obj, int len, int dumplen,
720 char *dst, int lim)
721 {
722 int i, j, i0, n = 0;
723 static char hex[] = "0123456789abcdef";
724 const char *p = obj; /* dump cursor */
725 char *o; /* output position */
726
727 #define P_HI(x) hex[((x) & 0xf0) >> 4]
728 #define P_LO(x) hex[((x) & 0xf)]
729 #define P_C(x) ((x) >= 0x20 && (x) <= 0x7e ? (x) : '.')
730 if (lim <= 0 || lim > len) {
731 lim = len;
732 }
733 if (dst == NULL) {
734 dst = sk_dump_buf;
735 lim = SK_DUMP_BUF_SIZE;
736 }
737 dumplen = MIN(len, dumplen);
738 o = dst;
739 n += scnprintf(o, lim, "%s 0x%llx len %d lim %d\n", label,
740 SK_KVA(p), len, lim);
741 o += strlen(o);
742 /* hexdump routine */
743 for (i = 0; i < dumplen;) {
744 n += scnprintf(o, lim - n, "%5d: ", i);
745 o += strlen(o);
746 memset(o, ' ', 48);
747 i0 = i;
748 for (j = 0; j < 16 && i < dumplen; i++, j++) {
749 o[j * 3] = P_HI(p[i]);
750 o[j * 3 + 1] = P_LO(p[i]);
751 }
752 i = i0;
753 for (j = 0; j < 16 && i < dumplen; i++, j++) {
754 o[j + 48] = P_C(p[i]);
755 }
756 o[j + 48] = '\n';
757 o += j + 49;
758 }
759 *o = '\0';
760 #undef P_HI
761 #undef P_LO
762 #undef P_C
763 return dst;
764 }
765
766 /*
767 * "Safe" variant of proc_name_address(), mean to be used only for logging.
768 */
769 const char *
sk_proc_name_address(struct proc * p)770 sk_proc_name_address(struct proc *p)
771 {
772 if (p == PROC_NULL) {
773 return "proc_null";
774 }
775
776 return proc_name_address(p);
777 }
778
779 /*
780 * "Safe" variant of proc_pid(), mean to be used only for logging.
781 */
782 int
sk_proc_pid(struct proc * p)783 sk_proc_pid(struct proc *p)
784 {
785 if (p == PROC_NULL) {
786 return -1;
787 }
788
789 return proc_pid(p);
790 }
791
792 const char *
sk_sa_ntop(struct sockaddr * sa,char * addr_str,size_t addr_strlen)793 sk_sa_ntop(struct sockaddr *sa, char *addr_str, size_t addr_strlen)
794 {
795 const char *str = NULL;
796
797 addr_str[0] = '\0';
798
799 switch (sa->sa_family) {
800 case AF_INET:
801 str = inet_ntop(AF_INET, &SIN(sa)->sin_addr.s_addr,
802 addr_str, (socklen_t)addr_strlen);
803 break;
804
805 case AF_INET6:
806 str = inet_ntop(AF_INET6, &SIN6(sa)->sin6_addr,
807 addr_str, (socklen_t)addr_strlen);
808 break;
809
810 default:
811 str = addr_str;
812 break;
813 }
814
815 return str;
816 }
817
818 const char *
sk_memstatus2str(uint32_t status)819 sk_memstatus2str(uint32_t status)
820 {
821 const char *str = NULL;
822
823 switch (status) {
824 case kMemorystatusInvalid:
825 str = "kMemorystatusInvalid";
826 break;
827
828 case kMemorystatusKilled:
829 str = "kMemorystatusKilled";
830 break;
831
832 case kMemorystatusKilledHiwat:
833 str = "kMemorystatusKilledHiwat";
834 break;
835
836 case kMemorystatusKilledVnodes:
837 str = "kMemorystatusKilledVnodes";
838 break;
839
840 case kMemorystatusKilledVMPageShortage:
841 str = "kMemorystatusKilledVMPageShortage";
842 break;
843
844 case kMemorystatusKilledProcThrashing:
845 str = "kMemorystatusKilledProcThrashing";
846 break;
847
848 case kMemorystatusKilledVMCompressorThrashing:
849 str = "kMemorystatusKilledVMCompressorThrashing";
850 break;
851
852 case kMemorystatusKilledVMCompressorSpaceShortage:
853 str = "kMemorystatusKilledVMCompressorSpaceShortage";
854 break;
855
856 case kMemorystatusKilledFCThrashing:
857 str = "kMemorystatusKilledFCThrashing";
858 break;
859
860 case kMemorystatusKilledPerProcessLimit:
861 str = "kMemorystatusKilledPerProcessLimit";
862 break;
863
864 case kMemorystatusKilledDiskSpaceShortage:
865 str = "kMemorystatusKilledDiskSpaceShortage";
866 break;
867
868 case kMemorystatusKilledIdleExit:
869 str = "kMemorystatusKilledIdleExit";
870 break;
871
872 case kMemorystatusKilledZoneMapExhaustion:
873 str = "kMemorystatusKilledZoneMapExhaustion";
874 break;
875
876 default:
877 str = "unknown";
878 break;
879 }
880
881 return str;
882 }
883 #endif /* SK_LOG */
884
885 bool
sk_sa_has_addr(struct sockaddr * sa)886 sk_sa_has_addr(struct sockaddr *sa)
887 {
888 switch (sa->sa_family) {
889 case AF_INET:
890 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
891 return SIN(sa)->sin_addr.s_addr != INADDR_ANY;
892 case AF_INET6:
893 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
894 return !IN6_IS_ADDR_UNSPECIFIED(&SIN6(sa)->sin6_addr);
895 default:
896 return false;
897 }
898 }
899
900 bool
sk_sa_has_port(struct sockaddr * sa)901 sk_sa_has_port(struct sockaddr *sa)
902 {
903 switch (sa->sa_family) {
904 case AF_INET:
905 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
906 return SIN(sa)->sin_port != 0;
907 case AF_INET6:
908 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
909 return SIN6(sa)->sin6_port != 0;
910 default:
911 return false;
912 }
913 }
914
915 /* returns port number in host byte order */
916 uint16_t
sk_sa_get_port(struct sockaddr * sa)917 sk_sa_get_port(struct sockaddr *sa)
918 {
919 switch (sa->sa_family) {
920 case AF_INET:
921 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
922 return ntohs(SIN(sa)->sin_port);
923 case AF_INET6:
924 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
925 return ntohs(SIN6(sa)->sin6_port);
926 default:
927 VERIFY(0);
928 /* NOTREACHED */
929 __builtin_unreachable();
930 }
931 }
932
933 void
skywalk_kill_process(struct proc * p,uint64_t reason_code)934 skywalk_kill_process(struct proc *p, uint64_t reason_code)
935 {
936 os_reason_t exit_reason = OS_REASON_NULL;
937
938 VERIFY(p != kernproc);
939
940 exit_reason = os_reason_create(OS_REASON_SKYWALK, reason_code);
941 if (exit_reason == OS_REASON_NULL) {
942 SK_ERR("%s(%d) unable to allocate memory for crash reason "
943 "0x%llX", sk_proc_name_address(p), sk_proc_pid(p),
944 reason_code);
945 } else {
946 exit_reason->osr_flags |= OS_REASON_FLAG_GENERATE_CRASH_REPORT;
947 SK_ERR("%s(%d) aborted for reason 0x%llX",
948 sk_proc_name_address(p), sk_proc_pid(p), reason_code);
949 }
950
951 psignal_try_thread_with_reason(p, current_thread(), SIGABRT,
952 exit_reason);
953 }
954
955 #if (DEVELOPMENT || DEBUG)
956 #define SK_MEMCMP_LEN 256 /* length of each section */
957 #define SK_MASK_MAXLEN 80 /* maximum mask length */
958
959 #define SK_MEMCMP_MASK_VERIFY(t, l, lr) do { \
960 _CASSERT(sizeof(t##_m) == SK_MASK_MAXLEN); \
961 if ((sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) != 0) ^ \
962 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, lr) != 0)) { \
963 panic_plain("\nbroken: " #t " using " \
964 "sk_memcmp_mask_" #l " at i=%d\n", i); \
965 /* NOTREACHED */ \
966 __builtin_unreachable(); \
967 } \
968 if ((sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) != 0) ^ \
969 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, lr) != 0)) { \
970 panic_plain("\nbroken: " #t " using " \
971 "sk_memcmp_mask_" #l "_scalar at i=%d\n", i); \
972 /* NOTREACHED */ \
973 __builtin_unreachable(); \
974 } \
975 } while (0)
976
977 #define SK_MEMCMP_MASK_MATCH_VERIFY(t, l) do { \
978 if (sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) != 0) { \
979 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
980 " mismatch (expected match) at i=%d s1=0x%x" \
981 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
982 /* NOTREACHED */ \
983 __builtin_unreachable(); \
984 } \
985 if (sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) != 0) { \
986 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
987 "_scalar mismatch (expected match) at i=%d s1=0x%x" \
988 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
989 /* NOTREACHED */ \
990 __builtin_unreachable(); \
991 } \
992 } while (0)
993
994 #define SK_MEMCMP_MASK_MISMATCH_VERIFY(t, l) do { \
995 if (sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) == 0) { \
996 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
997 " match (expected mismatch) at i=%d s1=0x%x" \
998 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
999 /* NOTREACHED */ \
1000 __builtin_unreachable(); \
1001 } \
1002 if (sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) == 0) { \
1003 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
1004 "_scalar match (expected mismatch) at i=%d " \
1005 "s1=0x%x s2=0x%x\n", i, hdr1[i], hdr2[i]); \
1006 /* NOTREACHED */ \
1007 __builtin_unreachable(); \
1008 } \
1009 } while (0)
1010
1011 #define SK_MEMCMP_BYTEMASK_VERIFY(t) do { \
1012 if ((sk_memcmp_mask(hdr1, hdr2, t##_m, i) != 0) ^ \
1013 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, i) != 0)) { \
1014 panic_plain("\nbroken: " #t " using " \
1015 "sk_memcmp_mask at i=%d\n", i); \
1016 /* NOTREACHED */ \
1017 __builtin_unreachable(); \
1018 } \
1019 if ((sk_memcmp_mask_scalar(hdr1, hdr2, t##_m, i) != 0) ^ \
1020 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, i) != 0)) { \
1021 panic_plain("\nbroken: " #t " using " \
1022 "sk_memcmp_mask_scalar at i=%d\n", i); \
1023 /* NOTREACHED */ \
1024 __builtin_unreachable(); \
1025 } \
1026 } while (0)
1027
1028 static inline int
skywalk_memcmp_mask_ref(const uint8_t * src1,const uint8_t * src2,const uint8_t * byte_mask,size_t n)1029 skywalk_memcmp_mask_ref(const uint8_t *src1, const uint8_t *src2,
1030 const uint8_t *byte_mask, size_t n)
1031 {
1032 uint32_t result = 0;
1033 for (size_t i = 0; i < n; i++) {
1034 result |= (src1[i] ^ src2[i]) & byte_mask[i];
1035 }
1036 return result;
1037 }
1038
1039 static void
skywalk_memcmp_mask_self_tests(void)1040 skywalk_memcmp_mask_self_tests(void)
1041 {
1042 static const uint8_t ipv4_m[] = {
1043 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0xff, 0xff,
1044 0xff, 0xff, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
1045 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1046 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1047 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1048 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1049 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1050 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1051 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1052 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1053 };
1054 static const uint8_t ipv6_m[] = {
1055 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1056 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1057 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1058 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1059 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1060 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1061 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1062 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1063 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1064 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1065 };
1066 static const uint8_t tcp_m[] = {
1067 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1068 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1069 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1070 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1071 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1072 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1073 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1074 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1075 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1076 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1077 };
1078 static const uint8_t ipv6_tcp_m[] = {
1079 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1080 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1081 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1082 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1083 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1084 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1085 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1086 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1087 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1088 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1089 };
1090 static const uint8_t udp_m[] = {
1091 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00,
1092 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1093 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1094 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1095 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1096 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1097 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1098 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1099 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1101 };
1102 static const uint8_t fk_all_m[] = {
1103 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1104 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1105 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1106 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1107 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1108 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1109 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1110 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1111 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1112 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1113 };
1114 static const uint8_t fk_t2_m[] = {
1115 0x0a, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
1116 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1117 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1118 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1119 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1120 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1121 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1122 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1123 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1124 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1125 };
1126 static const uint8_t fk_t3_m[] = {
1127 0x0f, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00,
1128 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1129 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1130 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1131 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1132 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1133 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1134 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1135 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1136 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1137 };
1138 static const uint8_t fk_t4_m[] = {
1139 0x2f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1140 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1141 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1142 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1143 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1144 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1145 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1146 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1147 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1148 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1149 };
1150 static const uint8_t fk_t5_m[] = {
1151 0x3f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1152 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1153 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1154 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1155 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1156 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1157 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1158 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1159 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1160 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1161 };
1162 static const uint8_t fk_i1_m[] = {
1163 0x02, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00,
1164 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1165 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1166 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1167 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1168 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1169 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1170 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1171 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1172 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1173 };
1174 static const uint8_t fk_i2_m[] = {
1175 0x07, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1176 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1177 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1178 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1179 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1180 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1181 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1182 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1183 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1184 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1185 };
1186 static const uint8_t fk_i3_m[] = {
1187 0x17, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1188 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1189 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1190 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1191 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1192 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1193 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1194 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1195 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1196 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1197 };
1198
1199 /* validate flow entry mask (2-tuple) */
1200 _CASSERT(FKMASK_2TUPLE == (FKMASK_PROTO | FKMASK_SPORT));
1201 VERIFY(fk_mask_2tuple.fk_mask == FKMASK_2TUPLE);
1202 VERIFY(fk_mask_2tuple.fk_ipver == 0);
1203 VERIFY(fk_mask_2tuple.fk_proto == 0xff);
1204 VERIFY(fk_mask_2tuple.fk_sport == 0xffff);
1205 VERIFY(fk_mask_2tuple.fk_dport == 0);
1206 VERIFY(fk_mask_2tuple.fk_src._addr64[0] == 0);
1207 VERIFY(fk_mask_2tuple.fk_src._addr64[1] == 0);
1208 VERIFY(fk_mask_2tuple.fk_dst._addr64[0] == 0);
1209 VERIFY(fk_mask_2tuple.fk_dst._addr64[1] == 0);
1210 VERIFY(fk_mask_2tuple.fk_pad[0] == 0);
1211
1212 _CASSERT(FKMASK_3TUPLE == (FKMASK_2TUPLE | FKMASK_IPVER | FKMASK_SRC));
1213 VERIFY(fk_mask_3tuple.fk_mask == FKMASK_3TUPLE);
1214 VERIFY(fk_mask_3tuple.fk_ipver == 0xff);
1215 VERIFY(fk_mask_3tuple.fk_proto == 0xff);
1216 VERIFY(fk_mask_3tuple.fk_sport == 0xffff);
1217 VERIFY(fk_mask_3tuple.fk_dport == 0);
1218 VERIFY(fk_mask_3tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1219 VERIFY(fk_mask_3tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1220 VERIFY(fk_mask_3tuple.fk_dst._addr64[0] == 0);
1221 VERIFY(fk_mask_3tuple.fk_dst._addr64[1] == 0);
1222 VERIFY(fk_mask_3tuple.fk_pad[0] == 0);
1223
1224 _CASSERT(FKMASK_4TUPLE == (FKMASK_3TUPLE | FKMASK_DPORT));
1225 VERIFY(fk_mask_4tuple.fk_mask == FKMASK_4TUPLE);
1226 VERIFY(fk_mask_4tuple.fk_ipver == 0xff);
1227 VERIFY(fk_mask_4tuple.fk_proto == 0xff);
1228 VERIFY(fk_mask_4tuple.fk_sport == 0xffff);
1229 VERIFY(fk_mask_4tuple.fk_dport == 0xffff);
1230 VERIFY(fk_mask_4tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1231 VERIFY(fk_mask_4tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1232 VERIFY(fk_mask_4tuple.fk_dst._addr64[0] == 0);
1233 VERIFY(fk_mask_4tuple.fk_dst._addr64[1] == 0);
1234 VERIFY(fk_mask_4tuple.fk_pad[0] == 0);
1235
1236 _CASSERT(FKMASK_5TUPLE == (FKMASK_4TUPLE | FKMASK_DST));
1237 VERIFY(fk_mask_5tuple.fk_mask == FKMASK_5TUPLE);
1238 VERIFY(fk_mask_5tuple.fk_ipver == 0xff);
1239 VERIFY(fk_mask_5tuple.fk_proto == 0xff);
1240 VERIFY(fk_mask_5tuple.fk_sport == 0xffff);
1241 VERIFY(fk_mask_5tuple.fk_dport == 0xffff);
1242 VERIFY(fk_mask_5tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1243 VERIFY(fk_mask_5tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1244 VERIFY(fk_mask_5tuple.fk_dst._addr64[0] == 0xffffffffffffffffULL);
1245 VERIFY(fk_mask_5tuple.fk_dst._addr64[1] == 0xffffffffffffffffULL);
1246 VERIFY(fk_mask_5tuple.fk_pad[0] == 0);
1247
1248 _CASSERT(FKMASK_IPFLOW1 == FKMASK_PROTO);
1249 VERIFY(fk_mask_ipflow1.fk_mask == FKMASK_IPFLOW1);
1250 VERIFY(fk_mask_ipflow1.fk_ipver == 0);
1251 VERIFY(fk_mask_ipflow1.fk_proto == 0xff);
1252 VERIFY(fk_mask_ipflow1.fk_sport == 0);
1253 VERIFY(fk_mask_ipflow1.fk_dport == 0);
1254 VERIFY(fk_mask_ipflow1.fk_src._addr64[0] == 0);
1255 VERIFY(fk_mask_ipflow1.fk_src._addr64[1] == 0);
1256 VERIFY(fk_mask_ipflow1.fk_dst._addr64[0] == 0);
1257 VERIFY(fk_mask_ipflow1.fk_dst._addr64[1] == 0);
1258 VERIFY(fk_mask_ipflow1.fk_pad[0] == 0);
1259
1260 _CASSERT(FKMASK_IPFLOW2 == (FKMASK_IPFLOW1 | FKMASK_IPVER | FKMASK_SRC));
1261 VERIFY(fk_mask_ipflow2.fk_mask == FKMASK_IPFLOW2);
1262 VERIFY(fk_mask_ipflow2.fk_ipver == 0xff);
1263 VERIFY(fk_mask_ipflow2.fk_proto == 0xff);
1264 VERIFY(fk_mask_ipflow2.fk_sport == 0);
1265 VERIFY(fk_mask_ipflow2.fk_dport == 0);
1266 VERIFY(fk_mask_ipflow2.fk_src._addr64[0] == 0xffffffffffffffffULL);
1267 VERIFY(fk_mask_ipflow2.fk_src._addr64[1] == 0xffffffffffffffffULL);
1268 VERIFY(fk_mask_ipflow2.fk_dst._addr64[0] == 0);
1269 VERIFY(fk_mask_ipflow2.fk_dst._addr64[1] == 0);
1270 VERIFY(fk_mask_ipflow2.fk_pad[0] == 0);
1271
1272 _CASSERT(FKMASK_IPFLOW3 == (FKMASK_IPFLOW2 | FKMASK_DST));
1273 VERIFY(fk_mask_ipflow3.fk_mask == FKMASK_IPFLOW3);
1274 VERIFY(fk_mask_ipflow3.fk_ipver == 0xff);
1275 VERIFY(fk_mask_ipflow3.fk_proto == 0xff);
1276 VERIFY(fk_mask_ipflow3.fk_sport == 0);
1277 VERIFY(fk_mask_ipflow3.fk_dport == 0);
1278 VERIFY(fk_mask_ipflow3.fk_src._addr64[0] == 0xffffffffffffffffULL);
1279 VERIFY(fk_mask_ipflow3.fk_src._addr64[1] == 0xffffffffffffffffULL);
1280 VERIFY(fk_mask_ipflow3.fk_dst._addr64[0] == 0xffffffffffffffffULL);
1281 VERIFY(fk_mask_ipflow3.fk_dst._addr64[1] == 0xffffffffffffffffULL);
1282 VERIFY(fk_mask_ipflow3.fk_pad[0] == 0);
1283
1284 VERIFY(sk_dump_buf != NULL);
1285
1286 /* reset sk_dump_buf */
1287 bzero(sk_dump_buf, SK_DUMP_BUF_SIZE);
1288
1289 /*
1290 * Utilize sk_dump_buf, by splitting it into 3 sections. Each
1291 * section begins on a 128-bit boundary, and is a multiple of
1292 * 64-bytes len. A section is SK_MEMCMP_LEN-bytes long,
1293 * which means we need at least 16+(3*SK_MEMCMP_LEN) bytes.
1294 *
1295 * 1st section is s1 -> (hdr1 aligned to 16-bytes)
1296 * 2nd section is s2 -> (hdr2 = hdr1 + SK_MEMCMP_LEN)
1297 * 3rd section is s3 -> (mask = hdr2 + SK_MEMCMP_LEN)
1298 */
1299 void *s1, *s2, *s3;
1300
1301 s1 = sk_dump_buf;
1302 if (!IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN)) {
1303 s1 = (void *)P2ROUNDUP(s1, SK_DUMP_BUF_ALIGN);
1304 }
1305 ASSERT(IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN));
1306 s2 = (void *)((uintptr_t)s1 + SK_MEMCMP_LEN);
1307 ASSERT(IS_P2ALIGNED(s2, SK_DUMP_BUF_ALIGN));
1308 s3 = (void *)((uintptr_t)s2 + SK_MEMCMP_LEN);
1309 ASSERT(IS_P2ALIGNED(s3, SK_DUMP_BUF_ALIGN));
1310
1311 uint8_t *hdr1 = s1;
1312 uint8_t *hdr2 = s2;
1313 uint8_t *byte_m = s3;
1314
1315 /* fill byte mask with random data */
1316 read_frandom(byte_m, SK_MEMCMP_LEN);
1317
1318 kprintf("Skywalk: memcmp mask ... ");
1319
1320 int i;
1321 for (i = 0; i < 80; i++) {
1322 hdr1[i] = 1;
1323 SK_MEMCMP_MASK_VERIFY(ipv4, 32, 20);
1324 SK_MEMCMP_MASK_VERIFY(ipv6, 64, 40);
1325 SK_MEMCMP_MASK_VERIFY(ipv6_tcp, 80, 64);
1326 SK_MEMCMP_MASK_VERIFY(tcp, 32, 24);
1327 SK_MEMCMP_MASK_VERIFY(udp, 16, 6);
1328 SK_MEMCMP_MASK_VERIFY(fk_all, 48, 48);
1329 SK_MEMCMP_MASK_VERIFY(fk_t2, 48, 48);
1330 SK_MEMCMP_MASK_VERIFY(fk_t3, 48, 48);
1331 SK_MEMCMP_MASK_VERIFY(fk_t4, 48, 48);
1332 SK_MEMCMP_MASK_VERIFY(fk_t5, 48, 48);
1333 SK_MEMCMP_MASK_VERIFY(fk_i1, 48, 48);
1334 SK_MEMCMP_MASK_VERIFY(fk_i2, 48, 48);
1335 SK_MEMCMP_MASK_VERIFY(fk_i3, 48, 48);
1336 hdr2[i] = 1;
1337 }
1338
1339 bzero(hdr1, SK_MEMCMP_LEN);
1340 bzero(hdr2, SK_MEMCMP_LEN);
1341
1342 /* re-fill byte mask with random data */
1343 read_frandom(byte_m, SK_MEMCMP_LEN);
1344
1345 for (i = 0; i < SK_MEMCMP_LEN; i++) {
1346 hdr1[i] = 1;
1347 SK_MEMCMP_BYTEMASK_VERIFY(byte);
1348 hdr2[i] = 1;
1349 }
1350
1351 /* fill hdr1 and hd2 with random data */
1352 read_frandom(hdr1, SK_MEMCMP_LEN);
1353 bcopy(hdr1, hdr2, SK_MEMCMP_LEN);
1354 memset(byte_m, 0xff, SK_MEMCMP_LEN);
1355
1356 for (i = 0; i < 80; i++) {
1357 uint8_t val = hdr2[i];
1358 uint8_t mval = byte_m[i];
1359
1360 while (hdr2[i] == hdr1[i] || hdr2[i] == 0) {
1361 uint8_t newval;
1362 read_frandom(&newval, sizeof(newval));
1363 hdr2[i] = newval;
1364 }
1365 if (i < 16) {
1366 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 16);
1367 } else if (i < 32) {
1368 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 32);
1369 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1370 } else if (i < 48) {
1371 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 48);
1372 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1373 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1374 } else if (i < 64) {
1375 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 64);
1376 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1377 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1378 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1379 } else if (i < 80) {
1380 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 80);
1381 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 64);
1382 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1383 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1384 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1385 }
1386 byte_m[i] = 0;
1387 if (i < 16) {
1388 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1389 } else if (i < 32) {
1390 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1391 } else if (i < 48) {
1392 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1393 } else if (i < 64) {
1394 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 64);
1395 } else if (i < 80) {
1396 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 80);
1397 }
1398 hdr2[i] = val;
1399 byte_m[i] = mval;
1400 }
1401
1402 kprintf("PASSED\n");
1403 }
1404
1405 #define SK_COPY_LEN 128 /* length of each section */
1406
1407 #define SK_COPY_PREPARE(t) do { \
1408 bzero(s2, SK_COPY_LEN); \
1409 bzero(s3, SK_COPY_LEN); \
1410 _s1 = s1; _s2 = s2; _s3 = s3; \
1411 kprintf("Skywalk: " #t " ... "); \
1412 } while (0)
1413
1414 #define SK_COPY_VERIFY(t) do { \
1415 if (_s1 != s1 || _s2 != s2 || _s3 != s3) { \
1416 panic_plain("\ninput registers clobbered: " #t "\n"); \
1417 /* NOTREACHED */ \
1418 __builtin_unreachable(); \
1419 } \
1420 if (bcmp(s2, s3, SK_COPY_LEN) != 0) { \
1421 panic_plain("\nbroken: " #t "\n"); \
1422 /* NOTREACHED */ \
1423 __builtin_unreachable(); \
1424 } else { \
1425 kprintf("PASSED\n"); \
1426 } \
1427 } while (0)
1428
1429 #define SK_ZERO_PREPARE(t) do { \
1430 bcopy(s1, s2, SK_COPY_LEN); \
1431 bcopy(s1, s3, SK_COPY_LEN); \
1432 _s1 = s1; _s2 = s2; _s3 = s3; \
1433 kprintf("Skywalk: " #t " ... "); \
1434 } while (0)
1435
1436 #define SK_ZERO_VERIFY(t) SK_COPY_VERIFY(t)
1437
1438 static void
skywalk_self_tests(void)1439 skywalk_self_tests(void)
1440 {
1441 void *s1, *s2, *s3;
1442 void *_s1, *_s2, *_s3;
1443
1444 VERIFY(sk_dump_buf != NULL);
1445
1446 /*
1447 * Utilize sk_dump_buf, by splitting it into 3 sections. Each
1448 * section begins on a 128-bit boundary, and is a multiple of
1449 * 64-bytes len. A section is 128-bytes long, which means we
1450 * need at least 16+(3*128) bytes.
1451 *
1452 * 1st section is source buffer full of random data;
1453 * 2nd section is reference target based on bcopy;
1454 * 3rd section is test target base on our stuff.
1455 */
1456 _CASSERT(SK_COPY_LEN != 0 && (SK_COPY_LEN % 128) == 0);
1457 _CASSERT((SK_COPY_LEN % 16) == 0);
1458 _CASSERT((SK_DUMP_BUF_ALIGN % 16) == 0);
1459 _CASSERT(SK_DUMP_BUF_SIZE >= (SK_DUMP_BUF_ALIGN + (SK_COPY_LEN * 3)));
1460
1461 s1 = sk_dump_buf;
1462 if (!IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN)) {
1463 s1 = (void *)P2ROUNDUP(s1, SK_DUMP_BUF_ALIGN);
1464 }
1465 ASSERT(IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN));
1466 s2 = (void *)((uintptr_t)s1 + SK_COPY_LEN);
1467 ASSERT(IS_P2ALIGNED(s2, SK_DUMP_BUF_ALIGN));
1468 s3 = (void *)((uintptr_t)s2 + SK_COPY_LEN);
1469 ASSERT(IS_P2ALIGNED(s3, SK_DUMP_BUF_ALIGN));
1470
1471 /* fill s1 with random data */
1472 read_frandom(s1, SK_COPY_LEN);
1473
1474 kprintf("Skywalk: running self-tests\n");
1475
1476 /* Copy 8-bytes, 64-bit aligned */
1477 SK_COPY_PREPARE(sk_copy64_8);
1478 bcopy(s1, s2, 8);
1479 sk_copy64_8((uint64_t *)s1, (uint64_t *)s3);
1480 SK_COPY_VERIFY(sk_copy64_8);
1481
1482 /* Copy 8-bytes, 32-bit aligned */
1483 SK_COPY_PREPARE(sk_copy32_8);
1484 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1485 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 8);
1486 sk_copy32_8((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1487 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1488 SK_COPY_VERIFY(sk_copy32_8);
1489
1490 /* Copy 16-bytes, 64-bit aligned */
1491 SK_COPY_PREPARE(sk_copy64_16);
1492 bcopy(s1, s2, 16);
1493 sk_copy64_16((uint64_t *)s1, (uint64_t *)s3);
1494 SK_COPY_VERIFY(sk_copy64_16);
1495
1496 /* Copy 16-bytes, 32-bit aligned */
1497 SK_COPY_PREPARE(sk_copy32_16);
1498 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1499 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 16);
1500 sk_copy32_16((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1501 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1502 SK_COPY_VERIFY(sk_copy32_16);
1503
1504 /* Copy 20-bytes, 64-bit aligned */
1505 SK_COPY_PREPARE(sk_copy64_20);
1506 bcopy(s1, s2, 20);
1507 sk_copy64_20((uint64_t *)s1, (uint64_t *)s3);
1508 SK_COPY_VERIFY(sk_copy64_20);
1509
1510 /* Copy 24-bytes, 64-bit aligned */
1511 SK_COPY_PREPARE(sk_copy64_24);
1512 bcopy(s1, s2, 24);
1513 sk_copy64_24((uint64_t *)s1, (uint64_t *)s3);
1514 SK_COPY_VERIFY(sk_copy64_24);
1515
1516 /* Copy 32-bytes, 64-bit aligned */
1517 SK_COPY_PREPARE(sk_copy64_32);
1518 bcopy(s1, s2, 32);
1519 sk_copy64_32((uint64_t *)s1, (uint64_t *)s3);
1520 SK_COPY_VERIFY(sk_copy64_32);
1521
1522 /* Copy 32-bytes, 32-bit aligned */
1523 SK_COPY_PREPARE(sk_copy32_32);
1524 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1525 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 32);
1526 sk_copy32_32((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1527 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1528 SK_COPY_VERIFY(sk_copy32_32);
1529
1530 /* Copy 40-bytes, 64-bit aligned */
1531 SK_COPY_PREPARE(sk_copy64_40);
1532 bcopy(s1, s2, 40);
1533 sk_copy64_40((uint64_t *)s1, (uint64_t *)s3);
1534 SK_COPY_VERIFY(sk_copy64_40);
1535
1536 /* Copy entire section in 64-bytes chunks, 64-bit aligned */
1537 SK_COPY_PREPARE(sk_copy64_64x);
1538 bcopy(s1, s2, SK_COPY_LEN);
1539 sk_copy64_64x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1540 SK_COPY_VERIFY(sk_copy64_64x);
1541
1542 /* Copy entire section in 32-bytes chunks, 64-bit aligned */
1543 SK_COPY_PREPARE(sk_copy64_32x);
1544 bcopy(s1, s2, SK_COPY_LEN);
1545 sk_copy64_32x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1546 SK_COPY_VERIFY(sk_copy64_32x);
1547
1548 /* Copy entire section in 8-bytes chunks, 64-bit aligned */
1549 SK_COPY_PREPARE(sk_copy64_8x);
1550 bcopy(s1, s2, SK_COPY_LEN);
1551 sk_copy64_8x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1552 SK_COPY_VERIFY(sk_copy64_8x);
1553
1554 /* Copy entire section in 4-bytes chunks, 64-bit aligned */
1555 SK_COPY_PREPARE(sk_copy64_4x);
1556 bcopy(s1, s2, SK_COPY_LEN);
1557 sk_copy64_4x((uint32_t *)s1, (uint32_t *)s3, SK_COPY_LEN);
1558 SK_COPY_VERIFY(sk_copy64_4x);
1559
1560 /*
1561 * Re-use sk_dump_buf for testing sk_zero, same principle as above.
1562 *
1563 * 1st section is source buffer full of random data;
1564 * 2nd section is reference target based on bzero;
1565 * 3rd section is test target base on our stuff.
1566 */
1567 SK_ZERO_PREPARE(sk_zero_16);
1568 bzero(s2, 16);
1569 sk_zero_16(s3);
1570 SK_ZERO_VERIFY(sk_zero_16);
1571
1572 SK_ZERO_PREPARE(sk_zero_32);
1573 bzero(s2, 32);
1574 sk_zero_32(s3);
1575 SK_ZERO_VERIFY(sk_zero_32);
1576
1577 SK_ZERO_PREPARE(sk_zero_48);
1578 bzero(s2, 48);
1579 sk_zero_48(s3);
1580 SK_ZERO_VERIFY(sk_zero_48);
1581
1582 SK_ZERO_PREPARE(sk_zero_128);
1583 bzero(s2, 128);
1584 sk_zero_128(s3);
1585 SK_ZERO_VERIFY(sk_zero_128);
1586
1587 /* Perform memcmp with mask self tests */
1588 skywalk_memcmp_mask_self_tests();
1589
1590 /* reset sk_dump_buf */
1591 bzero(sk_dump_buf, SK_DUMP_BUF_SIZE);
1592
1593 /* Keep packet trace code in sync with ariadne plist */
1594 _CASSERT(SK_KTRACE_AON_IF_STATS == 0x8100004);
1595
1596 _CASSERT(SK_KTRACE_FSW_DEV_RING_FLUSH == 0x8110004);
1597 _CASSERT(SK_KTRACE_FSW_USER_RING_FLUSH == 0x8110008);
1598 _CASSERT(SK_KTRACE_FSW_FLOW_TRACK_RTT == 0x8110010);
1599
1600 _CASSERT(SK_KTRACE_NETIF_RING_TX_REFILL == 0x8120004);
1601 _CASSERT(SK_KTRACE_NETIF_HOST_ENQUEUE == 0x8120008);
1602 _CASSERT(SK_KTRACE_NETIF_MIT_RX_INTR == 0x812000c);
1603 _CASSERT(SK_KTRACE_NETIF_COMMON_INTR == 0x8120010);
1604 _CASSERT(SK_KTRACE_NETIF_RX_NOTIFY_DEFAULT == 0x8120014);
1605 _CASSERT(SK_KTRACE_NETIF_RX_NOTIFY_FAST == 0x8120018);
1606
1607 _CASSERT(SK_KTRACE_CHANNEL_TX_REFILL == 0x8130004);
1608
1609 _CASSERT(SK_KTRACE_PKT_RX_DRV == 0x8140004);
1610 _CASSERT(SK_KTRACE_PKT_RX_FSW == 0x8140008);
1611 _CASSERT(SK_KTRACE_PKT_RX_CHN == 0x814000c);
1612 _CASSERT(SK_KTRACE_PKT_TX_FSW == 0x8140040);
1613 _CASSERT(SK_KTRACE_PKT_TX_AQM == 0x8140044);
1614 _CASSERT(SK_KTRACE_PKT_TX_DRV == 0x8140048);
1615 }
1616 #endif /* DEVELOPMENT || DEBUG */
1617