1 /*
2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <skywalk/os_skywalk_private.h>
30 #include <pexpert/pexpert.h> /* for PE_parse_boot_argn */
31 #include <sys/codesign.h> /* for csproc_get_platform_binary */
32 #include <sys/signalvar.h> /* for psignal_with_reason */
33 #include <sys/reason.h>
34 #include <sys/kern_memorystatus.h>
35 #if CONFIG_MACF
36 #include <security/mac_framework.h>
37 #endif /* CONFIG_MACF */
38
39 #ifndef htole16
40 #if BYTE_ORDER == LITTLE_ENDIAN
41 #define htole16(x) ((uint16_t)(x))
42 #else /* BYTE_ORDER != LITTLE_ENDIAN */
43 #define htole16(x) bswap16((x))
44 #endif /* BYTE_ORDER == LITTLE_ENDIAN */
45 #endif /* htole16 */
46
47 LCK_GRP_DECLARE(sk_lock_group, "sk_lock");
48 LCK_ATTR_DECLARE(sk_lock_attr, 0, 0);
49 LCK_MTX_DECLARE_ATTR(sk_lock, &sk_lock_group, &sk_lock_attr);
50
51 static void skywalk_fini(void);
52 static int sk_priv_chk(proc_t, kauth_cred_t, int);
53
54 static int __sk_inited = 0;
55 #if (DEVELOPMENT || DEBUG)
56 size_t sk_copy_thres = SK_COPY_THRES;
57 uint64_t sk_verbose;
58 #endif /* DEVELOPMENT || DEBUG */
59 uint32_t sk_debug;
60 uint64_t sk_features =
61 #if SKYWALK
62 SK_FEATURE_SKYWALK |
63 #endif
64 #if DEVELOPMENT
65 SK_FEATURE_DEVELOPMENT |
66 #endif
67 #if DEBUG
68 SK_FEATURE_DEBUG |
69 #endif
70 #if CONFIG_NEXUS_FLOWSWITCH
71 SK_FEATURE_NEXUS_FLOWSWITCH |
72 #endif
73 #if CONFIG_NEXUS_MONITOR
74 SK_FEATURE_NEXUS_MONITOR |
75 #endif
76 #if CONFIG_NEXUS_NETIF
77 SK_FEATURE_NEXUS_NETIF |
78 #endif
79 #if CONFIG_NEXUS_USER_PIPE
80 SK_FEATURE_NEXUS_USER_PIPE |
81 #endif
82 #if CONFIG_NEXUS_KERNEL_PIPE
83 SK_FEATURE_NEXUS_KERNEL_PIPE |
84 #endif
85 #if CONFIG_NEXUS_KERNEL_PIPE && (DEVELOPMENT || DEBUG)
86 SK_FEATURE_NEXUS_KERNEL_PIPE_LOOPBACK |
87 #endif
88 #if (DEVELOPMENT || DEBUG)
89 SK_FEATURE_DEV_OR_DEBUG |
90 #endif
91 0;
92
93 uint32_t sk_opp_defunct = 0; /* opportunistic defunct */
94
95 /* checksum offload is generic to any nexus (not specific to flowswitch) */
96 uint32_t sk_cksum_tx = 1; /* advertise outbound offload */
97 uint32_t sk_cksum_rx = 1; /* perform inbound checksum offload */
98
99 /* guard pages */
100 uint32_t sk_guard = 0; /* guard pages (0: disable) */
101 #define SK_GUARD_MIN 1 /* minimum # of guard pages */
102 #define SK_GUARD_MAX 4 /* maximum # of guard pages */
103 uint32_t sk_headguard_sz = SK_GUARD_MIN; /* # of leading guard pages */
104 uint32_t sk_tailguard_sz = SK_GUARD_MIN; /* # of trailing guard pages */
105
106 #if (DEVELOPMENT || DEBUG)
107 uint32_t sk_txring_sz = 0; /* flowswitch */
108 uint32_t sk_rxring_sz = 0; /* flowswitch */
109 uint32_t sk_net_txring_sz = 0; /* netif adapter */
110 uint32_t sk_net_rxring_sz = 0; /* netif adapter */
111 uint32_t sk_min_pool_size = 0; /* minimum packet pool size */
112 #endif /* !DEVELOPMENT && !DEBUG */
113
114 uint32_t sk_max_flows = NX_FLOWADV_DEFAULT;
115 uint32_t sk_fadv_nchunks; /* # of FO_FLOWADV_CHUNK in bitmap */
116 uint32_t sk_netif_compat_txmodel = NETIF_COMPAT_TXMODEL_DEFAULT;
117 uint32_t sk_netif_native_txmodel = NETIF_NATIVE_TXMODEL_DEFAULT;
118 /*
119 * Configures the RX aggregation logic for TCP in flowswitch.
120 * A non-zero value enables the aggregation logic, with the maximum
121 * aggregation length (in bytes) limited to this value.
122 */
123 uint32_t sk_fsw_rx_agg_tcp = 16384;
124
125 /*
126 * Forces the RX host path to use or not use aggregation, regardless of the
127 * existence of filters (see sk_fsw_rx_agg_tcp_host_t for valid values).
128 */
129 uint32_t sk_fsw_rx_agg_tcp_host = SK_FSW_RX_AGG_TCP_HOST_AUTO;
130
131 /*
132 * Configures the skywalk infrastructure for handling TCP TX aggregation.
133 * A non-zero value enables the support.
134 */
135 uint32_t sk_fsw_tx_agg_tcp = 1;
136 /*
137 * Configuration to limit the number of buffers for flowswitch VP channel.
138 */
139 uint32_t sk_fsw_max_bufs = 0;
140
141 /* list of interfaces that allow direct open from userspace */
142 #define SK_NETIF_DIRECT_MAX 8
143 char sk_netif_direct[SK_NETIF_DIRECT_MAX][IFXNAMSIZ];
144 uint32_t sk_netif_direct_cnt = 0;
145
146 uint16_t sk_tx_delay_qlen = 16; /* packets */
147 uint16_t sk_tx_delay_timeout = (1 * 1000); /* microseconds */
148
149 #define SK_NETIF_COMPAT_AUX_CELL_TX_RING_SZ 64
150 #define SK_NETIF_COMPAT_AUX_CELL_RX_RING_SZ 64
151 uint32_t sk_netif_compat_aux_cell_tx_ring_sz =
152 SK_NETIF_COMPAT_AUX_CELL_TX_RING_SZ;
153 uint32_t sk_netif_compat_aux_cell_rx_ring_sz =
154 SK_NETIF_COMPAT_AUX_CELL_RX_RING_SZ;
155
156 /* Wi-Fi Access Point */
157 #define SK_NETIF_COMPAT_WAP_TX_RING_SZ 128
158 #define SK_NETIF_COMPAT_WAP_RX_RING_SZ 128
159 uint32_t sk_netif_compat_wap_tx_ring_sz = SK_NETIF_COMPAT_WAP_TX_RING_SZ;
160 uint32_t sk_netif_compat_wap_rx_ring_sz = SK_NETIF_COMPAT_WAP_RX_RING_SZ;
161
162 /* AWDL */
163 #define SK_NETIF_COMPAT_AWDL_TX_RING_SZ 128
164 #define SK_NETIF_COMPAT_AWDL_RX_RING_SZ 128
165 uint32_t sk_netif_compat_awdl_tx_ring_sz = SK_NETIF_COMPAT_AWDL_TX_RING_SZ;
166 uint32_t sk_netif_compat_awdl_rx_ring_sz = SK_NETIF_COMPAT_AWDL_RX_RING_SZ;
167
168 /* Wi-Fi Infrastructure */
169 #define SK_NETIF_COMPAT_WIF_TX_RING_SZ 128
170 #define SK_NETIF_COMPAT_WIF_RX_RING_SZ 128
171 uint32_t sk_netif_compat_wif_tx_ring_sz = SK_NETIF_COMPAT_WIF_TX_RING_SZ;
172 uint32_t sk_netif_compat_wif_rx_ring_sz = SK_NETIF_COMPAT_WIF_RX_RING_SZ;
173
174 #define SK_NETIF_COMPAT_USB_ETH_TX_RING_SZ 128
175 #define SK_NETIF_COMPAT_USB_ETH_RX_RING_SZ 128
176 uint32_t sk_netif_compat_usb_eth_tx_ring_sz =
177 SK_NETIF_COMPAT_USB_ETH_TX_RING_SZ;
178 uint32_t sk_netif_compat_usb_eth_rx_ring_sz =
179 SK_NETIF_COMPAT_USB_ETH_RX_RING_SZ;
180
181 #define SK_NETIF_COMPAT_RX_MBQ_LIMIT 8192
182 int sk_netif_compat_rx_mbq_limit = SK_NETIF_COMPAT_RX_MBQ_LIMIT;
183
184 uint32_t sk_netif_tx_mit = SK_NETIF_MIT_AUTO;
185 uint32_t sk_netif_rx_mit = SK_NETIF_MIT_AUTO;
186 char sk_ll_prefix[IFNAMSIZ] = "llw";
187 uint32_t sk_rx_sync_packets = 1;
188 uint32_t sk_channel_buflet_alloc = 0;
189
190 SYSCTL_NODE(_kern, OID_AUTO, skywalk, CTLFLAG_RW | CTLFLAG_LOCKED,
191 0, "Skywalk parameters");
192 SYSCTL_NODE(_kern_skywalk, OID_AUTO, stats, CTLFLAG_RW | CTLFLAG_LOCKED,
193 0, "Skywalk statistics");
194
195 SYSCTL_OPAQUE(_kern_skywalk, OID_AUTO, features, CTLFLAG_RD | CTLFLAG_LOCKED,
196 &sk_features, sizeof(sk_features), "-", "Skywalk features");
197
198 #if (DEVELOPMENT || DEBUG)
199 SYSCTL_QUAD(_kern_skywalk, OID_AUTO, verbose, CTLFLAG_RW | CTLFLAG_LOCKED,
200 &sk_verbose, "Skywalk verbose mode");
201 SYSCTL_UINT(_kern_skywalk, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
202 &sk_debug, 0, "Skywalk debug mode");
203 SYSCTL_LONG(_kern_skywalk, OID_AUTO, sk_copy_thres, CTLFLAG_RW | CTLFLAG_LOCKED,
204 &sk_copy_thres, "Skywalk copy threshold");
205 static int __priv_check = 1;
206 SYSCTL_INT(_kern_skywalk, OID_AUTO, priv_check, CTLFLAG_RW | CTLFLAG_LOCKED,
207 &__priv_check, 0, "Skywalk privilege check");
208 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_opp_defunct, CTLFLAG_RW | CTLFLAG_LOCKED,
209 &sk_opp_defunct, 0, "Defunct opportunistically");
210 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_cksum_tx, CTLFLAG_RW | CTLFLAG_LOCKED,
211 &sk_cksum_tx, 0, "Advertise (and perform) outbound checksum offload");
212 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_cksum_rx, CTLFLAG_RW | CTLFLAG_LOCKED,
213 &sk_cksum_rx, 0, "Perform inbound checksum offload");
214 SYSCTL_UINT(_kern_skywalk, OID_AUTO, sk_rx_sync_packets, CTLFLAG_RW | CTLFLAG_LOCKED,
215 &sk_rx_sync_packets, 0, "Enable RX sync packets");
216 SYSCTL_UINT(_kern_skywalk, OID_AUTO, chan_buf_alloc,
217 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_channel_buflet_alloc, 0,
218 "channel buflet allocation (enable/disable)");
219 #endif /* !DEVELOPMENT && !DEBUG */
220
221 #if (DEVELOPMENT || DEBUG)
222 uint32_t sk_inject_error_rmask = 0x3;
223 SYSCTL_UINT(_kern_skywalk, OID_AUTO, inject_error_rmask,
224 CTLFLAG_RW | CTLFLAG_LOCKED, &sk_inject_error_rmask, 0x3, "");
225 #endif /* !DEVELOPMENT && !DEBUG */
226
227 #if (DEVELOPMENT || DEBUG)
228 static void skywalk_self_tests(void);
229 #endif /* (DEVELOPMENT || DEBUG) */
230
231 #define SKMEM_TAG_SYSCTL_BUF "com.apple.skywalk.sysctl_buf"
232 SKMEM_TAG_DEFINE(skmem_tag_sysctl_buf, SKMEM_TAG_SYSCTL_BUF);
233
234 #define SKMEM_TAG_OID "com.apple.skywalk.skoid"
235 SKMEM_TAG_DEFINE(skmem_tag_oid, SKMEM_TAG_OID);
236
237 #if (SK_LOG || DEVELOPMENT || DEBUG)
238 #define SKMEM_TAG_DUMP "com.apple.skywalk.dump"
239 static SKMEM_TAG_DEFINE(skmem_tag_dump, SKMEM_TAG_DUMP);
240
241 static char *sk_dump_buf;
242 #define SK_DUMP_BUF_SIZE 2048
243 #define SK_DUMP_BUF_ALIGN 16
244 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
245
246 __startup_func
247 void
__sk_tag_make(const struct sk_tag_spec * spec)248 __sk_tag_make(const struct sk_tag_spec *spec)
249 {
250 *spec->skt_var = kern_allocation_name_allocate(spec->skt_name, 0);
251 }
252
253 boolean_t
skywalk_netif_direct_enabled(void)254 skywalk_netif_direct_enabled(void)
255 {
256 return sk_netif_direct_cnt > 0;
257 }
258
259 boolean_t
skywalk_netif_direct_allowed(const char * ifname)260 skywalk_netif_direct_allowed(const char *ifname)
261 {
262 uint32_t i;
263
264 for (i = 0; i < sk_netif_direct_cnt; i++) {
265 if (strncmp(ifname, sk_netif_direct[i], IFXNAMSIZ) == 0) {
266 return TRUE;
267 }
268 }
269 return FALSE;
270 }
271
272 #if (DEVELOPMENT || DEBUG)
273 static void
parse_netif_direct(void)274 parse_netif_direct(void)
275 {
276 char buf[(IFXNAMSIZ + 1) * SK_NETIF_DIRECT_MAX];
277 size_t i, curr, len, iflen;
278
279 if (!PE_parse_boot_arg_str("sk_netif_direct", buf, sizeof(buf))) {
280 return;
281 }
282
283 curr = 0;
284 len = strlen(buf);
285 for (i = 0; i < len + 1 &&
286 sk_netif_direct_cnt < SK_NETIF_DIRECT_MAX; i++) {
287 if (buf[i] != ',' && buf[i] != '\0') {
288 continue;
289 }
290
291 buf[i] = '\0';
292 iflen = i - curr;
293 if (iflen > 0 && iflen < IFXNAMSIZ) {
294 (void) strlcpy(sk_netif_direct[sk_netif_direct_cnt],
295 buf + curr, IFXNAMSIZ);
296 sk_netif_direct_cnt++;
297 }
298 curr = i + 1;
299 }
300 }
301 #endif /* DEVELOPMENT || DEBUG */
302
303 static void
skywalk_fini(void)304 skywalk_fini(void)
305 {
306 SK_LOCK_ASSERT_HELD();
307
308 if (__sk_inited) {
309 #if (DEVELOPMENT || DEBUG)
310 skmem_test_fini();
311 cht_test_fini();
312 #endif /* (DEVELOPMENT || DEBUG) */
313 channel_fini();
314 nexus_fini();
315 skmem_fini();
316 flowidns_fini();
317
318 #if (SK_LOG || DEVELOPMENT || DEBUG)
319 if (sk_dump_buf != NULL) {
320 sk_free_data(sk_dump_buf, SK_DUMP_BUF_SIZE);
321 sk_dump_buf = NULL;
322 }
323 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
324
325 __sk_inited = 0;
326 }
327 }
328
329 int
skywalk_init(void)330 skywalk_init(void)
331 {
332 int error;
333
334 VERIFY(!__sk_inited);
335
336 _CASSERT(sizeof(kern_packet_t) == sizeof(uint64_t));
337 _CASSERT(sizeof(bitmap_t) == sizeof(uint64_t));
338
339 #if (DEVELOPMENT || DEBUG)
340 PE_parse_boot_argn("sk_debug", &sk_debug, sizeof(sk_debug));
341 PE_parse_boot_argn("sk_verbose", &sk_verbose, sizeof(sk_verbose));
342 (void) PE_parse_boot_argn("sk_opp_defunct", &sk_opp_defunct,
343 sizeof(sk_opp_defunct));
344 (void) PE_parse_boot_argn("sk_cksum_tx", &sk_cksum_tx,
345 sizeof(sk_cksum_tx));
346 (void) PE_parse_boot_argn("sk_cksum_rx", &sk_cksum_rx,
347 sizeof(sk_cksum_rx));
348 (void) PE_parse_boot_argn("sk_txring_sz", &sk_txring_sz,
349 sizeof(sk_txring_sz));
350 (void) PE_parse_boot_argn("sk_rxring_sz", &sk_rxring_sz,
351 sizeof(sk_rxring_sz));
352 (void) PE_parse_boot_argn("sk_net_txring_sz", &sk_net_txring_sz,
353 sizeof(sk_net_txring_sz));
354 (void) PE_parse_boot_argn("sk_net_rxring_sz", &sk_net_rxring_sz,
355 sizeof(sk_net_rxring_sz));
356 (void) PE_parse_boot_argn("sk_max_flows", &sk_max_flows,
357 sizeof(sk_max_flows));
358 (void) PE_parse_boot_argn("sk_native_txmodel", &sk_netif_native_txmodel,
359 sizeof(sk_netif_native_txmodel));
360 (void) PE_parse_boot_argn("sk_compat_txmodel", &sk_netif_compat_txmodel,
361 sizeof(sk_netif_compat_txmodel));
362 (void) PE_parse_boot_argn("sk_tx_delay_qlen", &sk_tx_delay_qlen,
363 sizeof(sk_tx_delay_qlen));
364 (void) PE_parse_boot_argn("sk_ts_delay_timeout", &sk_tx_delay_timeout,
365 sizeof(sk_tx_delay_timeout));
366 (void) PE_parse_boot_argn("sk_compat_aux_cell_tx_ring_sz",
367 &sk_netif_compat_aux_cell_tx_ring_sz,
368 sizeof(sk_netif_compat_aux_cell_tx_ring_sz));
369 (void) PE_parse_boot_argn("sk_compat_aux_cell_rx_ring_sz",
370 &sk_netif_compat_aux_cell_rx_ring_sz,
371 sizeof(sk_netif_compat_aux_cell_rx_ring_sz));
372 (void) PE_parse_boot_argn("sk_compat_wap_tx_ring_sz",
373 &sk_netif_compat_wap_tx_ring_sz,
374 sizeof(sk_netif_compat_wap_tx_ring_sz));
375 (void) PE_parse_boot_argn("sk_compat_wap_rx_ring_sz",
376 &sk_netif_compat_wap_rx_ring_sz,
377 sizeof(sk_netif_compat_wap_rx_ring_sz));
378 (void) PE_parse_boot_argn("sk_compat_awdl_tx_ring_sz",
379 &sk_netif_compat_awdl_tx_ring_sz,
380 sizeof(sk_netif_compat_awdl_tx_ring_sz));
381 (void) PE_parse_boot_argn("sk_compat_awdl_rx_ring_sz",
382 &sk_netif_compat_awdl_rx_ring_sz,
383 sizeof(sk_netif_compat_awdl_rx_ring_sz));
384 (void) PE_parse_boot_argn("sk_compat_wif_tx_ring_sz",
385 &sk_netif_compat_wif_tx_ring_sz,
386 sizeof(sk_netif_compat_wif_tx_ring_sz));
387 (void) PE_parse_boot_argn("sk_compat_wif_rx_ring_sz",
388 &sk_netif_compat_wif_rx_ring_sz,
389 sizeof(sk_netif_compat_wif_rx_ring_sz));
390 (void) PE_parse_boot_argn("sk_compat_usb_eth_tx_ring_sz",
391 &sk_netif_compat_usb_eth_tx_ring_sz,
392 sizeof(sk_netif_compat_usb_eth_tx_ring_sz));
393 (void) PE_parse_boot_argn("sk_compat_usb_eth_rx_ring_sz",
394 &sk_netif_compat_usb_eth_rx_ring_sz,
395 sizeof(sk_netif_compat_usb_eth_rx_ring_sz));
396 (void) PE_parse_boot_argn("sk_compat_rx_mbq_limit",
397 &sk_netif_compat_rx_mbq_limit, sizeof(sk_netif_compat_rx_mbq_limit));
398 (void) PE_parse_boot_argn("sk_netif_tx_mit",
399 &sk_netif_tx_mit, sizeof(sk_netif_tx_mit));
400 (void) PE_parse_boot_argn("sk_netif_rx_mit",
401 &sk_netif_rx_mit, sizeof(sk_netif_rx_mit));
402 (void) PE_parse_boot_arg_str("sk_ll_prefix", sk_ll_prefix,
403 sizeof(sk_ll_prefix));
404 parse_netif_direct();
405 (void) PE_parse_boot_argn("sk_fsw_rx_agg_tcp", &sk_fsw_rx_agg_tcp,
406 sizeof(sk_fsw_rx_agg_tcp));
407 (void) PE_parse_boot_argn("sk_fsw_tx_agg_tcp", &sk_fsw_tx_agg_tcp,
408 sizeof(sk_fsw_tx_agg_tcp));
409 (void) PE_parse_boot_argn("sk_fsw_max_bufs", &sk_fsw_max_bufs,
410 sizeof(sk_fsw_max_bufs));
411 (void) PE_parse_boot_argn("sk_rx_sync_packets", &sk_rx_sync_packets,
412 sizeof(sk_rx_sync_packets));
413 (void) PE_parse_boot_argn("sk_chan_buf_alloc", &sk_channel_buflet_alloc,
414 sizeof(sk_channel_buflet_alloc));
415 (void) PE_parse_boot_argn("sk_guard", &sk_guard, sizeof(sk_guard));
416 (void) PE_parse_boot_argn("sk_headguard_sz", &sk_headguard_sz,
417 sizeof(sk_headguard_sz));
418 (void) PE_parse_boot_argn("sk_tailguard_sz", &sk_tailguard_sz,
419 sizeof(sk_tailguard_sz));
420 (void) PE_parse_boot_argn("sk_min_pool_size", &sk_min_pool_size,
421 sizeof(sk_min_pool_size));
422 #endif /* DEVELOPMENT || DEBUG */
423
424 if (sk_max_flows == 0) {
425 sk_max_flows = NX_FLOWADV_DEFAULT;
426 } else if (sk_max_flows > NX_FLOWADV_MAX) {
427 sk_max_flows = NX_FLOWADV_MAX;
428 }
429
430 if (sk_netif_tx_mit > SK_NETIF_MIT_MAX) {
431 sk_netif_tx_mit = SK_NETIF_MIT_MAX;
432 }
433 if (sk_netif_rx_mit > SK_NETIF_MIT_MAX) {
434 sk_netif_rx_mit = SK_NETIF_MIT_MAX;
435 }
436
437 sk_fadv_nchunks = (uint32_t)P2ROUNDUP(sk_max_flows, FO_FLOWADV_CHUNK) /
438 FO_FLOWADV_CHUNK;
439
440 if (sk_guard) {
441 uint32_t sz;
442 /* leading guard page(s) */
443 if (sk_headguard_sz == 0) {
444 read_frandom(&sz, sizeof(sz));
445 sk_headguard_sz = (sz % (SK_GUARD_MAX + 1));
446 } else if (sk_headguard_sz > SK_GUARD_MAX) {
447 sk_headguard_sz = SK_GUARD_MAX;
448 }
449 if (sk_headguard_sz < SK_GUARD_MIN) {
450 sk_headguard_sz = SK_GUARD_MIN;
451 }
452 /* trailing guard page(s) */
453 if (sk_tailguard_sz == 0) {
454 read_frandom(&sz, sizeof(sz));
455 sk_tailguard_sz = (sz % (SK_GUARD_MAX + 1));
456 } else if (sk_tailguard_sz > SK_GUARD_MAX) {
457 sk_tailguard_sz = SK_GUARD_MAX;
458 }
459 if (sk_tailguard_sz < SK_GUARD_MIN) {
460 sk_tailguard_sz = SK_GUARD_MIN;
461 }
462 } else {
463 sk_headguard_sz = sk_tailguard_sz = SK_GUARD_MIN;
464 }
465 ASSERT(sk_headguard_sz >= SK_GUARD_MIN);
466 ASSERT(sk_headguard_sz <= SK_GUARD_MAX);
467 ASSERT(sk_tailguard_sz >= SK_GUARD_MIN);
468 ASSERT(sk_tailguard_sz <= SK_GUARD_MAX);
469
470 __sk_inited = 1;
471
472 SK_LOCK();
473 skmem_init();
474 error = nexus_init();
475 if (error == 0) {
476 error = channel_init();
477 }
478 if (error != 0) {
479 skywalk_fini();
480 }
481 SK_UNLOCK();
482
483 if (error == 0) {
484 #if (SK_LOG || DEVELOPMENT || DEBUG)
485 /* allocate space for sk_dump_buf */
486 sk_dump_buf = sk_alloc_data(SK_DUMP_BUF_SIZE, Z_WAITOK | Z_NOFAIL,
487 skmem_tag_dump);
488 #endif /* (SK_LOG || DEVELOPMENT || DEBUG) */
489
490 netns_init();
491 protons_init();
492 flowidns_init();
493
494 #if (DEVELOPMENT || DEBUG)
495 skywalk_self_tests();
496 skmem_test_init();
497 cht_test_init();
498 #endif /* DEVELOPMENT || DEBUG */
499 }
500
501 return error;
502 }
503
504 /*
505 * csproc_get_platform_binary() returns non-zero if the process is platform
506 * code, which means that it is considered part of the Operating System.
507 * On iOS, that means it's contained in the trust cache or a loaded one.
508 * On macOS, everything signed by B&I is currently platform code, but the
509 * policy in general is subject to change. Thus this is an approximate.
510 */
511 boolean_t
skywalk_check_platform_binary(proc_t p)512 skywalk_check_platform_binary(proc_t p)
513 {
514 return (csproc_get_platform_binary(p) == 0) ? FALSE : TRUE;
515 }
516
517 static int
sk_priv_chk(proc_t p,kauth_cred_t cred,int priv)518 sk_priv_chk(proc_t p, kauth_cred_t cred, int priv)
519 {
520 #pragma unused(p)
521 int ret = priv_check_cred(cred, priv, 0);
522
523 #if (DEVELOPMENT || DEBUG)
524 if (ret != 0) {
525 const char *pstr;
526
527 switch (priv) {
528 case PRIV_SKYWALK_REGISTER_USER_PIPE:
529 pstr = "com.apple.private.skywalk.register-user-pipe";
530 break;
531
532 case PRIV_SKYWALK_REGISTER_KERNEL_PIPE:
533 pstr = "com.apple.private.skywalk.register-kernel-pipe";
534 break;
535
536 case PRIV_SKYWALK_REGISTER_NET_IF:
537 pstr = "com.apple.private.skywalk.register-net-if";
538 break;
539
540 case PRIV_SKYWALK_REGISTER_FLOW_SWITCH:
541 pstr = "com.apple.private.skywalk.register-flow-switch";
542 break;
543
544 case PRIV_SKYWALK_OBSERVE_ALL:
545 pstr = "com.apple.private.skywalk.observe-all";
546 break;
547
548 case PRIV_SKYWALK_OBSERVE_STATS:
549 pstr = "com.apple.private.skywalk.observe-stats";
550 break;
551
552 case PRIV_SKYWALK_LOW_LATENCY_CHANNEL:
553 pstr = "com.apple.private.skywalk.low-latency-channel";
554 break;
555
556 default:
557 pstr = "unknown";
558 break;
559 }
560
561 #if SK_LOG
562 if (__priv_check) {
563 SK_DF(SK_VERB_PRIV, "%s(%d) insufficient privilege %d "
564 "(\"%s\") err %d", sk_proc_name_address(p),
565 sk_proc_pid(p), priv, pstr, ret);
566 } else {
567 SK_DF(SK_VERB_PRIV, "%s(%d) IGNORING missing privilege "
568 "%d (\"%s\") err %d", sk_proc_name_address(p),
569 sk_proc_pid(p), priv, pstr, ret);
570 }
571 #endif /* SK_LOG */
572
573 /* ignore privilege check failures if requested */
574 if (!__priv_check) {
575 ret = 0;
576 }
577 }
578 #endif /* !DEVELOPMENT && !DEBUG */
579
580 return ret;
581 }
582
583 int
skywalk_priv_check_cred(proc_t p,kauth_cred_t cred,int priv)584 skywalk_priv_check_cred(proc_t p, kauth_cred_t cred, int priv)
585 {
586 return sk_priv_chk(p, cred, priv);
587 }
588
589 int
skywalk_priv_check_proc_cred(proc_t p,int priv)590 skywalk_priv_check_proc_cred(proc_t p, int priv)
591 {
592 int ret;
593 kauth_cred_t cred = kauth_cred_proc_ref(p);
594 ret = sk_priv_chk(p, cred, priv);
595 kauth_cred_unref(&cred);
596
597 return ret;
598 }
599
600 #if CONFIG_MACF
601 int
skywalk_mac_system_check_proc_cred(proc_t p,const char * info_type)602 skywalk_mac_system_check_proc_cred(proc_t p, const char *info_type)
603 {
604 int ret;
605 kauth_cred_t cred = kauth_cred_proc_ref(p);
606 ret = mac_system_check_info(cred, info_type);
607 kauth_cred_unref(&cred);
608
609 return ret;
610 }
611 #endif /* CONFIG_MACF */
612
613 /*
614 * Scan thru the list of privileges needed before we allow someone
615 * to open a handle to the Nexus controller. This should be done
616 * at nxctl_create() time, and additional privilege check specific
617 * to the operation (e.g. register, etc.) should be done afterwards.
618 */
619 int
skywalk_nxctl_check_privileges(proc_t p,kauth_cred_t cred)620 skywalk_nxctl_check_privileges(proc_t p, kauth_cred_t cred)
621 {
622 int ret = 0;
623
624 if (p == kernproc) {
625 goto done;
626 }
627
628 do {
629 /*
630 * Check for observe-{stats,all} entitlements first
631 * before the rest, to account for nexus controller
632 * clients that don't need anything but statistics;
633 * it would help quiesce sandbox violation warnings.
634 */
635 if ((ret = sk_priv_chk(p, cred,
636 PRIV_SKYWALK_OBSERVE_STATS)) == 0) {
637 break;
638 }
639 if ((ret = sk_priv_chk(p, cred,
640 PRIV_SKYWALK_OBSERVE_ALL)) == 0) {
641 break;
642 }
643 if ((ret = sk_priv_chk(p, cred,
644 PRIV_SKYWALK_REGISTER_USER_PIPE)) == 0) {
645 break;
646 }
647 if ((ret = sk_priv_chk(p, cred,
648 PRIV_SKYWALK_REGISTER_KERNEL_PIPE)) == 0) {
649 break;
650 }
651 if ((ret = sk_priv_chk(p, cred,
652 PRIV_SKYWALK_REGISTER_NET_IF)) == 0) {
653 break;
654 }
655 if ((ret = sk_priv_chk(p, cred,
656 PRIV_SKYWALK_REGISTER_FLOW_SWITCH)) == 0) {
657 break;
658 }
659 /* none set, so too bad */
660 ret = EPERM;
661 } while (0);
662
663 #if (DEVELOPMENT || DEBUG)
664 if (ret != 0) {
665 SK_ERR("%s(%d) insufficient privilege to open nexus controller "
666 "err %d", sk_proc_name_address(p), sk_proc_pid(p), ret);
667 }
668 #endif /* !DEVELOPMENT && !DEBUG */
669 done:
670 return ret;
671 }
672
673 void
sk_gen_guard_id(boolean_t isch,const uuid_t uuid,guardid_t * guard)674 sk_gen_guard_id(boolean_t isch, const uuid_t uuid, guardid_t *guard)
675 {
676 #define GUARD_CH_SIG 0x4348 /* 'CH' */
677 #define GUARD_NX_SIG 0x4e58 /* 'NX' */
678 union {
679 uint8_t _u8[8];
680 uint16_t _u16[4];
681 uint64_t _u64;
682 } __u;
683
684 read_random(&__u._u16[0], sizeof(uint16_t));
685 bcopy(uuid, (void *)&__u._u16[1], sizeof(uint16_t));
686 __u._u16[2] = htole16(isch ? GUARD_CH_SIG : GUARD_NX_SIG);
687 __u._u16[3] = htole16(0x534b); /* 'SK' */
688 VERIFY(__u._u64 != 0);
689
690 bzero(guard, sizeof(*guard));
691 bcopy((void *)&__u._u64, guard, MIN(sizeof(*guard),
692 sizeof(uint64_t)));
693 }
694
695
696 extern const char *
sk_uuid_unparse(const uuid_t uu,uuid_string_t out)697 sk_uuid_unparse(const uuid_t uu, uuid_string_t out)
698 {
699 uuid_unparse_upper(uu, out);
700 return (const char *)out;
701 }
702
703 #if SK_LOG
704 /*
705 * packet-dump function, user-supplied or static buffer.
706 * The destination buffer must be at least 30+4*len
707 *
708 * @param p
709 * buffer to be dumped.
710 * @param len
711 * buffer's total length.
712 * @param dumplen
713 * length to be dumped.
714 * @param dst
715 * destination char buffer. sk_dump_buf would be used if dst is NULL.
716 * @param lim
717 * destination char buffer max length. Not used if dst is NULL.
718 *
719 */
720 const char *
sk_dump(const char * label,const void * obj,int len,int dumplen,char * dst,int lim)721 sk_dump(const char *label, const void *obj, int len, int dumplen,
722 char *dst, int lim)
723 {
724 int i, j, i0, n = 0;
725 static char hex[] = "0123456789abcdef";
726 const char *p = obj; /* dump cursor */
727 char *o; /* output position */
728
729 #define P_HI(x) hex[((x) & 0xf0) >> 4]
730 #define P_LO(x) hex[((x) & 0xf)]
731 #define P_C(x) ((x) >= 0x20 && (x) <= 0x7e ? (x) : '.')
732 if (lim <= 0 || lim > len) {
733 lim = len;
734 }
735 if (dst == NULL) {
736 dst = sk_dump_buf;
737 lim = SK_DUMP_BUF_SIZE;
738 }
739 dumplen = MIN(len, dumplen);
740 o = dst;
741 n += scnprintf(o, lim, "%s 0x%llx len %d lim %d\n", label,
742 SK_KVA(p), len, lim);
743 o += strlen(o);
744 /* hexdump routine */
745 for (i = 0; i < dumplen;) {
746 n += scnprintf(o, lim - n, "%5d: ", i);
747 o += strlen(o);
748 memset(o, ' ', 48);
749 i0 = i;
750 for (j = 0; j < 16 && i < dumplen; i++, j++) {
751 o[j * 3] = P_HI(p[i]);
752 o[j * 3 + 1] = P_LO(p[i]);
753 }
754 i = i0;
755 for (j = 0; j < 16 && i < dumplen; i++, j++) {
756 o[j + 48] = P_C(p[i]);
757 }
758 o[j + 48] = '\n';
759 o += j + 49;
760 }
761 *o = '\0';
762 #undef P_HI
763 #undef P_LO
764 #undef P_C
765 return dst;
766 }
767
768 /*
769 * "Safe" variant of proc_name_address(), mean to be used only for logging.
770 */
771 const char *
sk_proc_name_address(struct proc * p)772 sk_proc_name_address(struct proc *p)
773 {
774 if (p == PROC_NULL) {
775 return "proc_null";
776 }
777
778 return proc_name_address(p);
779 }
780
781 /*
782 * "Safe" variant of proc_pid(), mean to be used only for logging.
783 */
784 int
sk_proc_pid(struct proc * p)785 sk_proc_pid(struct proc *p)
786 {
787 if (p == PROC_NULL) {
788 return -1;
789 }
790
791 return proc_pid(p);
792 }
793
794 const char *
sk_sa_ntop(struct sockaddr * sa,char * addr_str,size_t addr_strlen)795 sk_sa_ntop(struct sockaddr *sa, char *addr_str, size_t addr_strlen)
796 {
797 const char *str = NULL;
798
799 addr_str[0] = '\0';
800
801 switch (sa->sa_family) {
802 case AF_INET:
803 str = inet_ntop(AF_INET, &SIN(sa)->sin_addr.s_addr,
804 addr_str, (socklen_t)addr_strlen);
805 break;
806
807 case AF_INET6:
808 str = inet_ntop(AF_INET6, &SIN6(sa)->sin6_addr,
809 addr_str, (socklen_t)addr_strlen);
810 break;
811
812 default:
813 str = addr_str;
814 break;
815 }
816
817 return str;
818 }
819
820 const char *
sk_memstatus2str(uint32_t status)821 sk_memstatus2str(uint32_t status)
822 {
823 const char *str = NULL;
824
825 switch (status) {
826 case kMemorystatusInvalid:
827 str = "kMemorystatusInvalid";
828 break;
829
830 case kMemorystatusKilled:
831 str = "kMemorystatusKilled";
832 break;
833
834 case kMemorystatusKilledHiwat:
835 str = "kMemorystatusKilledHiwat";
836 break;
837
838 case kMemorystatusKilledVnodes:
839 str = "kMemorystatusKilledVnodes";
840 break;
841
842 case kMemorystatusKilledVMPageShortage:
843 str = "kMemorystatusKilledVMPageShortage";
844 break;
845
846 case kMemorystatusKilledProcThrashing:
847 str = "kMemorystatusKilledProcThrashing";
848 break;
849
850 case kMemorystatusKilledVMCompressorThrashing:
851 str = "kMemorystatusKilledVMCompressorThrashing";
852 break;
853
854 case kMemorystatusKilledVMCompressorSpaceShortage:
855 str = "kMemorystatusKilledVMCompressorSpaceShortage";
856 break;
857
858 case kMemorystatusKilledFCThrashing:
859 str = "kMemorystatusKilledFCThrashing";
860 break;
861
862 case kMemorystatusKilledPerProcessLimit:
863 str = "kMemorystatusKilledPerProcessLimit";
864 break;
865
866 case kMemorystatusKilledDiskSpaceShortage:
867 str = "kMemorystatusKilledDiskSpaceShortage";
868 break;
869
870 case kMemorystatusKilledIdleExit:
871 str = "kMemorystatusKilledIdleExit";
872 break;
873
874 case kMemorystatusKilledZoneMapExhaustion:
875 str = "kMemorystatusKilledZoneMapExhaustion";
876 break;
877
878 default:
879 str = "unknown";
880 break;
881 }
882
883 return str;
884 }
885 #endif /* SK_LOG */
886
887 bool
sk_sa_has_addr(struct sockaddr * sa)888 sk_sa_has_addr(struct sockaddr *sa)
889 {
890 switch (sa->sa_family) {
891 case AF_INET:
892 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
893 return SIN(sa)->sin_addr.s_addr != INADDR_ANY;
894 case AF_INET6:
895 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
896 return !IN6_IS_ADDR_UNSPECIFIED(&SIN6(sa)->sin6_addr);
897 default:
898 return false;
899 }
900 }
901
902 bool
sk_sa_has_port(struct sockaddr * sa)903 sk_sa_has_port(struct sockaddr *sa)
904 {
905 switch (sa->sa_family) {
906 case AF_INET:
907 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
908 return SIN(sa)->sin_port != 0;
909 case AF_INET6:
910 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
911 return SIN6(sa)->sin6_port != 0;
912 default:
913 return false;
914 }
915 }
916
917 /* returns port number in host byte order */
918 uint16_t
sk_sa_get_port(struct sockaddr * sa)919 sk_sa_get_port(struct sockaddr *sa)
920 {
921 switch (sa->sa_family) {
922 case AF_INET:
923 ASSERT(sa->sa_len == sizeof(struct sockaddr_in));
924 return ntohs(SIN(sa)->sin_port);
925 case AF_INET6:
926 ASSERT(sa->sa_len == sizeof(struct sockaddr_in6));
927 return ntohs(SIN6(sa)->sin6_port);
928 default:
929 VERIFY(0);
930 /* NOTREACHED */
931 __builtin_unreachable();
932 }
933 }
934
935 void
skywalk_kill_process(struct proc * p,uint64_t reason_code)936 skywalk_kill_process(struct proc *p, uint64_t reason_code)
937 {
938 os_reason_t exit_reason = OS_REASON_NULL;
939
940 VERIFY(p != kernproc);
941
942 exit_reason = os_reason_create(OS_REASON_SKYWALK, reason_code);
943 if (exit_reason == OS_REASON_NULL) {
944 SK_ERR("%s(%d) unable to allocate memory for crash reason "
945 "0x%llX", sk_proc_name_address(p), sk_proc_pid(p),
946 reason_code);
947 } else {
948 exit_reason->osr_flags |= OS_REASON_FLAG_GENERATE_CRASH_REPORT;
949 SK_ERR("%s(%d) aborted for reason 0x%llX",
950 sk_proc_name_address(p), sk_proc_pid(p), reason_code);
951 }
952
953 psignal_try_thread_with_reason(p, current_thread(), SIGABRT,
954 exit_reason);
955 }
956
957 #if (DEVELOPMENT || DEBUG)
958 #define SK_MEMCMP_LEN 256 /* length of each section */
959 #define SK_MASK_MAXLEN 80 /* maximum mask length */
960
961 #define SK_MEMCMP_MASK_VERIFY(t, l, lr) do { \
962 _CASSERT(sizeof(t##_m) == SK_MASK_MAXLEN); \
963 if ((sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) != 0) ^ \
964 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, lr) != 0)) { \
965 panic_plain("\nbroken: " #t " using " \
966 "sk_memcmp_mask_" #l " at i=%d\n", i); \
967 /* NOTREACHED */ \
968 __builtin_unreachable(); \
969 } \
970 if ((sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) != 0) ^ \
971 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, lr) != 0)) { \
972 panic_plain("\nbroken: " #t " using " \
973 "sk_memcmp_mask_" #l "_scalar at i=%d\n", i); \
974 /* NOTREACHED */ \
975 __builtin_unreachable(); \
976 } \
977 } while (0)
978
979 #define SK_MEMCMP_MASK_MATCH_VERIFY(t, l) do { \
980 if (sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) != 0) { \
981 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
982 " mismatch (expected match) at i=%d s1=0x%x" \
983 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
984 /* NOTREACHED */ \
985 __builtin_unreachable(); \
986 } \
987 if (sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) != 0) { \
988 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
989 "_scalar mismatch (expected match) at i=%d s1=0x%x" \
990 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
991 /* NOTREACHED */ \
992 __builtin_unreachable(); \
993 } \
994 } while (0)
995
996 #define SK_MEMCMP_MASK_MISMATCH_VERIFY(t, l) do { \
997 if (sk_memcmp_mask_##l##B(hdr1, hdr2, t##_m) == 0) { \
998 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
999 " match (expected mismatch) at i=%d s1=0x%x" \
1000 " s2=0x%x\n", i, hdr1[i], hdr2[i]); \
1001 /* NOTREACHED */ \
1002 __builtin_unreachable(); \
1003 } \
1004 if (sk_memcmp_mask_##l##B##_scalar(hdr1, hdr2, t##_m) == 0) { \
1005 panic_plain("\nbroken: " #t " using sk_memcmp_mask_" #l \
1006 "_scalar match (expected mismatch) at i=%d " \
1007 "s1=0x%x s2=0x%x\n", i, hdr1[i], hdr2[i]); \
1008 /* NOTREACHED */ \
1009 __builtin_unreachable(); \
1010 } \
1011 } while (0)
1012
1013 #define SK_MEMCMP_BYTEMASK_VERIFY(t) do { \
1014 if ((sk_memcmp_mask(hdr1, hdr2, t##_m, i) != 0) ^ \
1015 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, i) != 0)) { \
1016 panic_plain("\nbroken: " #t " using " \
1017 "sk_memcmp_mask at i=%d\n", i); \
1018 /* NOTREACHED */ \
1019 __builtin_unreachable(); \
1020 } \
1021 if ((sk_memcmp_mask_scalar(hdr1, hdr2, t##_m, i) != 0) ^ \
1022 (skywalk_memcmp_mask_ref(hdr1, hdr2, t##_m, i) != 0)) { \
1023 panic_plain("\nbroken: " #t " using " \
1024 "sk_memcmp_mask_scalar at i=%d\n", i); \
1025 /* NOTREACHED */ \
1026 __builtin_unreachable(); \
1027 } \
1028 } while (0)
1029
1030 static inline int
skywalk_memcmp_mask_ref(const uint8_t * src1,const uint8_t * src2,const uint8_t * byte_mask,size_t n)1031 skywalk_memcmp_mask_ref(const uint8_t *src1, const uint8_t *src2,
1032 const uint8_t *byte_mask, size_t n)
1033 {
1034 uint32_t result = 0;
1035 for (size_t i = 0; i < n; i++) {
1036 result |= (src1[i] ^ src2[i]) & byte_mask[i];
1037 }
1038 return result;
1039 }
1040
1041 static void
skywalk_memcmp_mask_self_tests(void)1042 skywalk_memcmp_mask_self_tests(void)
1043 {
1044 static const uint8_t ipv4_m[] = {
1045 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0xff, 0xff,
1046 0xff, 0xff, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff,
1047 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1048 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1049 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1050 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1051 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1052 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1053 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1054 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1055 };
1056 static const uint8_t ipv6_m[] = {
1057 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1058 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1059 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1060 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1061 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1062 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1063 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1064 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1065 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1066 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1067 };
1068 static const uint8_t tcp_m[] = {
1069 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1070 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1071 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1072 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1073 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1074 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1075 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1076 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1077 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1078 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1079 };
1080 static const uint8_t ipv6_tcp_m[] = {
1081 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1082 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1083 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1084 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1085 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1086 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1087 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1088 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1089 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1090 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1091 };
1092 static const uint8_t udp_m[] = {
1093 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00,
1094 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1095 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1096 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1097 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1098 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1099 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1103 };
1104 static const uint8_t fk_all_m[] = {
1105 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1106 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1107 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1108 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1109 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1110 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1111 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1112 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1113 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1114 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1115 };
1116 static const uint8_t fk_t2_m[] = {
1117 0x0a, 0x00, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
1118 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1119 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1120 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1121 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1122 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1123 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1124 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1125 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1126 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1127 };
1128 static const uint8_t fk_t3_m[] = {
1129 0x0f, 0x00, 0xff, 0xff, 0xff, 0xff, 0x00, 0x00,
1130 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1131 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1132 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1133 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1134 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1135 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1136 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1137 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1138 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1139 };
1140 static const uint8_t fk_t4_m[] = {
1141 0x2f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1142 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1143 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1144 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1145 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1146 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1147 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1148 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1149 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1150 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1151 };
1152 static const uint8_t fk_t5_m[] = {
1153 0x3f, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1154 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1155 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1156 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1157 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1158 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1159 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1160 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1161 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1162 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1163 };
1164 static const uint8_t fk_i1_m[] = {
1165 0x02, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0x00,
1166 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1167 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1168 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1169 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1170 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1171 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1172 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1173 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1174 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1175 };
1176 static const uint8_t fk_i2_m[] = {
1177 0x07, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1178 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1179 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1180 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1181 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1182 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1183 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1184 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1185 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1186 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1187 };
1188 static const uint8_t fk_i3_m[] = {
1189 0x17, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00,
1190 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1191 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1192 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1193 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
1194 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1195 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1196 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1197 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1198 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1199 };
1200
1201 /* validate flow entry mask (2-tuple) */
1202 _CASSERT(FKMASK_2TUPLE == (FKMASK_PROTO | FKMASK_SPORT));
1203 VERIFY(fk_mask_2tuple.fk_mask == FKMASK_2TUPLE);
1204 VERIFY(fk_mask_2tuple.fk_ipver == 0);
1205 VERIFY(fk_mask_2tuple.fk_proto == 0xff);
1206 VERIFY(fk_mask_2tuple.fk_sport == 0xffff);
1207 VERIFY(fk_mask_2tuple.fk_dport == 0);
1208 VERIFY(fk_mask_2tuple.fk_src._addr64[0] == 0);
1209 VERIFY(fk_mask_2tuple.fk_src._addr64[1] == 0);
1210 VERIFY(fk_mask_2tuple.fk_dst._addr64[0] == 0);
1211 VERIFY(fk_mask_2tuple.fk_dst._addr64[1] == 0);
1212 VERIFY(fk_mask_2tuple.fk_pad[0] == 0);
1213
1214 _CASSERT(FKMASK_3TUPLE == (FKMASK_2TUPLE | FKMASK_IPVER | FKMASK_SRC));
1215 VERIFY(fk_mask_3tuple.fk_mask == FKMASK_3TUPLE);
1216 VERIFY(fk_mask_3tuple.fk_ipver == 0xff);
1217 VERIFY(fk_mask_3tuple.fk_proto == 0xff);
1218 VERIFY(fk_mask_3tuple.fk_sport == 0xffff);
1219 VERIFY(fk_mask_3tuple.fk_dport == 0);
1220 VERIFY(fk_mask_3tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1221 VERIFY(fk_mask_3tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1222 VERIFY(fk_mask_3tuple.fk_dst._addr64[0] == 0);
1223 VERIFY(fk_mask_3tuple.fk_dst._addr64[1] == 0);
1224 VERIFY(fk_mask_3tuple.fk_pad[0] == 0);
1225
1226 _CASSERT(FKMASK_4TUPLE == (FKMASK_3TUPLE | FKMASK_DPORT));
1227 VERIFY(fk_mask_4tuple.fk_mask == FKMASK_4TUPLE);
1228 VERIFY(fk_mask_4tuple.fk_ipver == 0xff);
1229 VERIFY(fk_mask_4tuple.fk_proto == 0xff);
1230 VERIFY(fk_mask_4tuple.fk_sport == 0xffff);
1231 VERIFY(fk_mask_4tuple.fk_dport == 0xffff);
1232 VERIFY(fk_mask_4tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1233 VERIFY(fk_mask_4tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1234 VERIFY(fk_mask_4tuple.fk_dst._addr64[0] == 0);
1235 VERIFY(fk_mask_4tuple.fk_dst._addr64[1] == 0);
1236 VERIFY(fk_mask_4tuple.fk_pad[0] == 0);
1237
1238 _CASSERT(FKMASK_5TUPLE == (FKMASK_4TUPLE | FKMASK_DST));
1239 VERIFY(fk_mask_5tuple.fk_mask == FKMASK_5TUPLE);
1240 VERIFY(fk_mask_5tuple.fk_ipver == 0xff);
1241 VERIFY(fk_mask_5tuple.fk_proto == 0xff);
1242 VERIFY(fk_mask_5tuple.fk_sport == 0xffff);
1243 VERIFY(fk_mask_5tuple.fk_dport == 0xffff);
1244 VERIFY(fk_mask_5tuple.fk_src._addr64[0] == 0xffffffffffffffffULL);
1245 VERIFY(fk_mask_5tuple.fk_src._addr64[1] == 0xffffffffffffffffULL);
1246 VERIFY(fk_mask_5tuple.fk_dst._addr64[0] == 0xffffffffffffffffULL);
1247 VERIFY(fk_mask_5tuple.fk_dst._addr64[1] == 0xffffffffffffffffULL);
1248 VERIFY(fk_mask_5tuple.fk_pad[0] == 0);
1249
1250 _CASSERT(FKMASK_IPFLOW1 == FKMASK_PROTO);
1251 VERIFY(fk_mask_ipflow1.fk_mask == FKMASK_IPFLOW1);
1252 VERIFY(fk_mask_ipflow1.fk_ipver == 0);
1253 VERIFY(fk_mask_ipflow1.fk_proto == 0xff);
1254 VERIFY(fk_mask_ipflow1.fk_sport == 0);
1255 VERIFY(fk_mask_ipflow1.fk_dport == 0);
1256 VERIFY(fk_mask_ipflow1.fk_src._addr64[0] == 0);
1257 VERIFY(fk_mask_ipflow1.fk_src._addr64[1] == 0);
1258 VERIFY(fk_mask_ipflow1.fk_dst._addr64[0] == 0);
1259 VERIFY(fk_mask_ipflow1.fk_dst._addr64[1] == 0);
1260 VERIFY(fk_mask_ipflow1.fk_pad[0] == 0);
1261
1262 _CASSERT(FKMASK_IPFLOW2 == (FKMASK_IPFLOW1 | FKMASK_IPVER | FKMASK_SRC));
1263 VERIFY(fk_mask_ipflow2.fk_mask == FKMASK_IPFLOW2);
1264 VERIFY(fk_mask_ipflow2.fk_ipver == 0xff);
1265 VERIFY(fk_mask_ipflow2.fk_proto == 0xff);
1266 VERIFY(fk_mask_ipflow2.fk_sport == 0);
1267 VERIFY(fk_mask_ipflow2.fk_dport == 0);
1268 VERIFY(fk_mask_ipflow2.fk_src._addr64[0] == 0xffffffffffffffffULL);
1269 VERIFY(fk_mask_ipflow2.fk_src._addr64[1] == 0xffffffffffffffffULL);
1270 VERIFY(fk_mask_ipflow2.fk_dst._addr64[0] == 0);
1271 VERIFY(fk_mask_ipflow2.fk_dst._addr64[1] == 0);
1272 VERIFY(fk_mask_ipflow2.fk_pad[0] == 0);
1273
1274 _CASSERT(FKMASK_IPFLOW3 == (FKMASK_IPFLOW2 | FKMASK_DST));
1275 VERIFY(fk_mask_ipflow3.fk_mask == FKMASK_IPFLOW3);
1276 VERIFY(fk_mask_ipflow3.fk_ipver == 0xff);
1277 VERIFY(fk_mask_ipflow3.fk_proto == 0xff);
1278 VERIFY(fk_mask_ipflow3.fk_sport == 0);
1279 VERIFY(fk_mask_ipflow3.fk_dport == 0);
1280 VERIFY(fk_mask_ipflow3.fk_src._addr64[0] == 0xffffffffffffffffULL);
1281 VERIFY(fk_mask_ipflow3.fk_src._addr64[1] == 0xffffffffffffffffULL);
1282 VERIFY(fk_mask_ipflow3.fk_dst._addr64[0] == 0xffffffffffffffffULL);
1283 VERIFY(fk_mask_ipflow3.fk_dst._addr64[1] == 0xffffffffffffffffULL);
1284 VERIFY(fk_mask_ipflow3.fk_pad[0] == 0);
1285
1286 VERIFY(sk_dump_buf != NULL);
1287
1288 /* reset sk_dump_buf */
1289 bzero(sk_dump_buf, SK_DUMP_BUF_SIZE);
1290
1291 /*
1292 * Utilize sk_dump_buf, by splitting it into 3 sections. Each
1293 * section begins on a 128-bit boundary, and is a multiple of
1294 * 64-bytes len. A section is SK_MEMCMP_LEN-bytes long,
1295 * which means we need at least 16+(3*SK_MEMCMP_LEN) bytes.
1296 *
1297 * 1st section is s1 -> (hdr1 aligned to 16-bytes)
1298 * 2nd section is s2 -> (hdr2 = hdr1 + SK_MEMCMP_LEN)
1299 * 3rd section is s3 -> (mask = hdr2 + SK_MEMCMP_LEN)
1300 */
1301 void *s1, *s2, *s3;
1302
1303 s1 = sk_dump_buf;
1304 if (!IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN)) {
1305 s1 = (void *)P2ROUNDUP(s1, SK_DUMP_BUF_ALIGN);
1306 }
1307 ASSERT(IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN));
1308 s2 = (void *)((uintptr_t)s1 + SK_MEMCMP_LEN);
1309 ASSERT(IS_P2ALIGNED(s2, SK_DUMP_BUF_ALIGN));
1310 s3 = (void *)((uintptr_t)s2 + SK_MEMCMP_LEN);
1311 ASSERT(IS_P2ALIGNED(s3, SK_DUMP_BUF_ALIGN));
1312
1313 uint8_t *hdr1 = s1;
1314 uint8_t *hdr2 = s2;
1315 uint8_t *byte_m = s3;
1316
1317 /* fill byte mask with random data */
1318 read_frandom(byte_m, SK_MEMCMP_LEN);
1319
1320 kprintf("Skywalk: memcmp mask ... ");
1321
1322 int i;
1323 for (i = 0; i < 80; i++) {
1324 hdr1[i] = 1;
1325 SK_MEMCMP_MASK_VERIFY(ipv4, 32, 20);
1326 SK_MEMCMP_MASK_VERIFY(ipv6, 64, 40);
1327 SK_MEMCMP_MASK_VERIFY(ipv6_tcp, 80, 64);
1328 SK_MEMCMP_MASK_VERIFY(tcp, 32, 24);
1329 SK_MEMCMP_MASK_VERIFY(udp, 16, 6);
1330 SK_MEMCMP_MASK_VERIFY(fk_all, 48, 48);
1331 SK_MEMCMP_MASK_VERIFY(fk_t2, 48, 48);
1332 SK_MEMCMP_MASK_VERIFY(fk_t3, 48, 48);
1333 SK_MEMCMP_MASK_VERIFY(fk_t4, 48, 48);
1334 SK_MEMCMP_MASK_VERIFY(fk_t5, 48, 48);
1335 SK_MEMCMP_MASK_VERIFY(fk_i1, 48, 48);
1336 SK_MEMCMP_MASK_VERIFY(fk_i2, 48, 48);
1337 SK_MEMCMP_MASK_VERIFY(fk_i3, 48, 48);
1338 hdr2[i] = 1;
1339 }
1340
1341 bzero(hdr1, SK_MEMCMP_LEN);
1342 bzero(hdr2, SK_MEMCMP_LEN);
1343
1344 /* re-fill byte mask with random data */
1345 read_frandom(byte_m, SK_MEMCMP_LEN);
1346
1347 for (i = 0; i < SK_MEMCMP_LEN; i++) {
1348 hdr1[i] = 1;
1349 SK_MEMCMP_BYTEMASK_VERIFY(byte);
1350 hdr2[i] = 1;
1351 }
1352
1353 /* fill hdr1 and hd2 with random data */
1354 read_frandom(hdr1, SK_MEMCMP_LEN);
1355 bcopy(hdr1, hdr2, SK_MEMCMP_LEN);
1356 memset(byte_m, 0xff, SK_MEMCMP_LEN);
1357
1358 for (i = 0; i < 80; i++) {
1359 uint8_t val = hdr2[i];
1360 uint8_t mval = byte_m[i];
1361
1362 while (hdr2[i] == hdr1[i] || hdr2[i] == 0) {
1363 uint8_t newval;
1364 read_frandom(&newval, sizeof(newval));
1365 hdr2[i] = newval;
1366 }
1367 if (i < 16) {
1368 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 16);
1369 } else if (i < 32) {
1370 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 32);
1371 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1372 } else if (i < 48) {
1373 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 48);
1374 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1375 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1376 } else if (i < 64) {
1377 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 64);
1378 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1379 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1380 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1381 } else if (i < 80) {
1382 SK_MEMCMP_MASK_MISMATCH_VERIFY(byte, 80);
1383 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 64);
1384 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1385 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1386 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1387 }
1388 byte_m[i] = 0;
1389 if (i < 16) {
1390 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 16);
1391 } else if (i < 32) {
1392 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 32);
1393 } else if (i < 48) {
1394 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 48);
1395 } else if (i < 64) {
1396 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 64);
1397 } else if (i < 80) {
1398 SK_MEMCMP_MASK_MATCH_VERIFY(byte, 80);
1399 }
1400 hdr2[i] = val;
1401 byte_m[i] = mval;
1402 }
1403
1404 kprintf("PASSED\n");
1405 }
1406
1407 #define SK_COPY_LEN 128 /* length of each section */
1408
1409 #define SK_COPY_PREPARE(t) do { \
1410 bzero(s2, SK_COPY_LEN); \
1411 bzero(s3, SK_COPY_LEN); \
1412 _s1 = s1; _s2 = s2; _s3 = s3; \
1413 kprintf("Skywalk: " #t " ... "); \
1414 } while (0)
1415
1416 #define SK_COPY_VERIFY(t) do { \
1417 if (_s1 != s1 || _s2 != s2 || _s3 != s3) { \
1418 panic_plain("\ninput registers clobbered: " #t "\n"); \
1419 /* NOTREACHED */ \
1420 __builtin_unreachable(); \
1421 } \
1422 if (bcmp(s2, s3, SK_COPY_LEN) != 0) { \
1423 panic_plain("\nbroken: " #t "\n"); \
1424 /* NOTREACHED */ \
1425 __builtin_unreachable(); \
1426 } else { \
1427 kprintf("PASSED\n"); \
1428 } \
1429 } while (0)
1430
1431 #define SK_ZERO_PREPARE(t) do { \
1432 bcopy(s1, s2, SK_COPY_LEN); \
1433 bcopy(s1, s3, SK_COPY_LEN); \
1434 _s1 = s1; _s2 = s2; _s3 = s3; \
1435 kprintf("Skywalk: " #t " ... "); \
1436 } while (0)
1437
1438 #define SK_ZERO_VERIFY(t) SK_COPY_VERIFY(t)
1439
1440 static void
skywalk_self_tests(void)1441 skywalk_self_tests(void)
1442 {
1443 void *s1, *s2, *s3;
1444 void *_s1, *_s2, *_s3;
1445
1446 VERIFY(sk_dump_buf != NULL);
1447
1448 /*
1449 * Utilize sk_dump_buf, by splitting it into 3 sections. Each
1450 * section begins on a 128-bit boundary, and is a multiple of
1451 * 64-bytes len. A section is 128-bytes long, which means we
1452 * need at least 16+(3*128) bytes.
1453 *
1454 * 1st section is source buffer full of random data;
1455 * 2nd section is reference target based on bcopy;
1456 * 3rd section is test target base on our stuff.
1457 */
1458 _CASSERT(SK_COPY_LEN != 0 && (SK_COPY_LEN % 128) == 0);
1459 _CASSERT((SK_COPY_LEN % 16) == 0);
1460 _CASSERT((SK_DUMP_BUF_ALIGN % 16) == 0);
1461 _CASSERT(SK_DUMP_BUF_SIZE >= (SK_DUMP_BUF_ALIGN + (SK_COPY_LEN * 3)));
1462
1463 s1 = sk_dump_buf;
1464 if (!IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN)) {
1465 s1 = (void *)P2ROUNDUP(s1, SK_DUMP_BUF_ALIGN);
1466 }
1467 ASSERT(IS_P2ALIGNED(s1, SK_DUMP_BUF_ALIGN));
1468 s2 = (void *)((uintptr_t)s1 + SK_COPY_LEN);
1469 ASSERT(IS_P2ALIGNED(s2, SK_DUMP_BUF_ALIGN));
1470 s3 = (void *)((uintptr_t)s2 + SK_COPY_LEN);
1471 ASSERT(IS_P2ALIGNED(s3, SK_DUMP_BUF_ALIGN));
1472
1473 /* fill s1 with random data */
1474 read_frandom(s1, SK_COPY_LEN);
1475
1476 kprintf("Skywalk: running self-tests\n");
1477
1478 /* Copy 8-bytes, 64-bit aligned */
1479 SK_COPY_PREPARE(sk_copy64_8);
1480 bcopy(s1, s2, 8);
1481 sk_copy64_8((uint64_t *)s1, (uint64_t *)s3);
1482 SK_COPY_VERIFY(sk_copy64_8);
1483
1484 /* Copy 8-bytes, 32-bit aligned */
1485 SK_COPY_PREPARE(sk_copy32_8);
1486 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1487 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 8);
1488 sk_copy32_8((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1489 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1490 SK_COPY_VERIFY(sk_copy32_8);
1491
1492 /* Copy 16-bytes, 64-bit aligned */
1493 SK_COPY_PREPARE(sk_copy64_16);
1494 bcopy(s1, s2, 16);
1495 sk_copy64_16((uint64_t *)s1, (uint64_t *)s3);
1496 SK_COPY_VERIFY(sk_copy64_16);
1497
1498 /* Copy 16-bytes, 32-bit aligned */
1499 SK_COPY_PREPARE(sk_copy32_16);
1500 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1501 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 16);
1502 sk_copy32_16((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1503 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1504 SK_COPY_VERIFY(sk_copy32_16);
1505
1506 /* Copy 20-bytes, 64-bit aligned */
1507 SK_COPY_PREPARE(sk_copy64_20);
1508 bcopy(s1, s2, 20);
1509 sk_copy64_20((uint64_t *)s1, (uint64_t *)s3);
1510 SK_COPY_VERIFY(sk_copy64_20);
1511
1512 /* Copy 24-bytes, 64-bit aligned */
1513 SK_COPY_PREPARE(sk_copy64_24);
1514 bcopy(s1, s2, 24);
1515 sk_copy64_24((uint64_t *)s1, (uint64_t *)s3);
1516 SK_COPY_VERIFY(sk_copy64_24);
1517
1518 /* Copy 32-bytes, 64-bit aligned */
1519 SK_COPY_PREPARE(sk_copy64_32);
1520 bcopy(s1, s2, 32);
1521 sk_copy64_32((uint64_t *)s1, (uint64_t *)s3);
1522 SK_COPY_VERIFY(sk_copy64_32);
1523
1524 /* Copy 32-bytes, 32-bit aligned */
1525 SK_COPY_PREPARE(sk_copy32_32);
1526 bcopy((void *)((uintptr_t)s1 + sizeof(uint32_t)),
1527 (void *)((uintptr_t)s2 + sizeof(uint32_t)), 32);
1528 sk_copy32_32((uint32_t *)((uintptr_t)s1 + sizeof(uint32_t)),
1529 (uint32_t *)((uintptr_t)s3 + sizeof(uint32_t)));
1530 SK_COPY_VERIFY(sk_copy32_32);
1531
1532 /* Copy 40-bytes, 64-bit aligned */
1533 SK_COPY_PREPARE(sk_copy64_40);
1534 bcopy(s1, s2, 40);
1535 sk_copy64_40((uint64_t *)s1, (uint64_t *)s3);
1536 SK_COPY_VERIFY(sk_copy64_40);
1537
1538 /* Copy entire section in 64-bytes chunks, 64-bit aligned */
1539 SK_COPY_PREPARE(sk_copy64_64x);
1540 bcopy(s1, s2, SK_COPY_LEN);
1541 sk_copy64_64x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1542 SK_COPY_VERIFY(sk_copy64_64x);
1543
1544 /* Copy entire section in 32-bytes chunks, 64-bit aligned */
1545 SK_COPY_PREPARE(sk_copy64_32x);
1546 bcopy(s1, s2, SK_COPY_LEN);
1547 sk_copy64_32x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1548 SK_COPY_VERIFY(sk_copy64_32x);
1549
1550 /* Copy entire section in 8-bytes chunks, 64-bit aligned */
1551 SK_COPY_PREPARE(sk_copy64_8x);
1552 bcopy(s1, s2, SK_COPY_LEN);
1553 sk_copy64_8x((uint64_t *)s1, (uint64_t *)s3, SK_COPY_LEN);
1554 SK_COPY_VERIFY(sk_copy64_8x);
1555
1556 /* Copy entire section in 4-bytes chunks, 64-bit aligned */
1557 SK_COPY_PREPARE(sk_copy64_4x);
1558 bcopy(s1, s2, SK_COPY_LEN);
1559 sk_copy64_4x((uint32_t *)s1, (uint32_t *)s3, SK_COPY_LEN);
1560 SK_COPY_VERIFY(sk_copy64_4x);
1561
1562 /*
1563 * Re-use sk_dump_buf for testing sk_zero, same principle as above.
1564 *
1565 * 1st section is source buffer full of random data;
1566 * 2nd section is reference target based on bzero;
1567 * 3rd section is test target base on our stuff.
1568 */
1569 SK_ZERO_PREPARE(sk_zero_16);
1570 bzero(s2, 16);
1571 sk_zero_16(s3);
1572 SK_ZERO_VERIFY(sk_zero_16);
1573
1574 SK_ZERO_PREPARE(sk_zero_32);
1575 bzero(s2, 32);
1576 sk_zero_32(s3);
1577 SK_ZERO_VERIFY(sk_zero_32);
1578
1579 SK_ZERO_PREPARE(sk_zero_48);
1580 bzero(s2, 48);
1581 sk_zero_48(s3);
1582 SK_ZERO_VERIFY(sk_zero_48);
1583
1584 SK_ZERO_PREPARE(sk_zero_128);
1585 bzero(s2, 128);
1586 sk_zero_128(s3);
1587 SK_ZERO_VERIFY(sk_zero_128);
1588
1589 /* Perform memcmp with mask self tests */
1590 skywalk_memcmp_mask_self_tests();
1591
1592 /* reset sk_dump_buf */
1593 bzero(sk_dump_buf, SK_DUMP_BUF_SIZE);
1594
1595 /* Keep packet trace code in sync with ariadne plist */
1596 _CASSERT(SK_KTRACE_AON_IF_STATS == 0x8100004);
1597
1598 _CASSERT(SK_KTRACE_FSW_DEV_RING_FLUSH == 0x8110004);
1599 _CASSERT(SK_KTRACE_FSW_USER_RING_FLUSH == 0x8110008);
1600 _CASSERT(SK_KTRACE_FSW_FLOW_TRACK_RTT == 0x8110010);
1601
1602 _CASSERT(SK_KTRACE_NETIF_RING_TX_REFILL == 0x8120004);
1603 _CASSERT(SK_KTRACE_NETIF_HOST_ENQUEUE == 0x8120008);
1604 _CASSERT(SK_KTRACE_NETIF_MIT_RX_INTR == 0x812000c);
1605 _CASSERT(SK_KTRACE_NETIF_COMMON_INTR == 0x8120010);
1606 _CASSERT(SK_KTRACE_NETIF_RX_NOTIFY_DEFAULT == 0x8120014);
1607 _CASSERT(SK_KTRACE_NETIF_RX_NOTIFY_FAST == 0x8120018);
1608
1609 _CASSERT(SK_KTRACE_CHANNEL_TX_REFILL == 0x8130004);
1610
1611 _CASSERT(SK_KTRACE_PKT_RX_DRV == 0x8140004);
1612 _CASSERT(SK_KTRACE_PKT_RX_FSW == 0x8140008);
1613 _CASSERT(SK_KTRACE_PKT_RX_CHN == 0x814000c);
1614 _CASSERT(SK_KTRACE_PKT_TX_FSW == 0x8140040);
1615 _CASSERT(SK_KTRACE_PKT_TX_AQM == 0x8140044);
1616 _CASSERT(SK_KTRACE_PKT_TX_DRV == 0x8140048);
1617 }
1618 #endif /* DEVELOPMENT || DEBUG */
1619