1 /* 2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 #ifndef _SKYWALK_OS_SKYWALK_PRIVATE_H 29 #define _SKYWALK_OS_SKYWALK_PRIVATE_H 30 31 #if defined(PRIVATE) || defined(XNU_KERNEL_PRIVATE) 32 /* 33 * This file contains private interfaces for Skywalk, and should not 34 * be included by code external to xnu kernel or libsystem_kernel. 35 * The only exception to this is skywalk_cmds for the internal tools. 36 */ 37 38 /* branch prediction helpers */ 39 #include <sys/cdefs.h> 40 #define SK_ALIGN64_CASSERT(type, field) \ 41 _CASSERT((__builtin_offsetof(type, field) % sizeof (uint64_t)) == 0) 42 43 #if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE) 44 enum { 45 SK_FEATURE_SKYWALK = 1ULL << 0, 46 SK_FEATURE_DEVELOPMENT = 1ULL << 1, 47 SK_FEATURE_DEBUG = 1ULL << 2, 48 SK_FEATURE_NEXUS_FLOWSWITCH = 1ULL << 3, 49 SK_FEATURE_NEXUS_MONITOR = 1ULL << 4, 50 SK_FEATURE_NEXUS_NETIF = 1ULL << 5, 51 SK_FEATURE_NEXUS_USER_PIPE = 1ULL << 6, 52 SK_FEATURE_NEXUS_KERNEL_PIPE = 1ULL << 7, 53 SK_FEATURE_NEXUS_KERNEL_PIPE_LOOPBACK = 1ULL << 8, 54 SK_FEATURE_DEV_OR_DEBUG = 1ULL << 9, 55 SK_FEATURE_NETNS = 1ULL << 10, 56 SK_FEATURE_PROTONS = 1ULL << 11, 57 }; 58 #endif /* !KERNEL || BSD_KERNEL_PRIVATE */ 59 60 /* valid flags for if_attach_nx */ 61 #define IF_ATTACH_NX_NETIF_COMPAT 0x01 /* create compat netif */ 62 #define IF_ATTACH_NX_FLOWSWITCH 0x02 /* enable flowswitch */ 63 #define IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT 0x04 /* enable fsw TCP/UDP netagent */ 64 #define IF_ATTACH_NX_NETIF_NETAGENT 0x08 /* enable netif netagent */ 65 #define IF_ATTACH_NX_NETIF_ALL 0x10 /* don't restrict netif */ 66 #define IF_ATTACH_NX_FSW_IP_NETAGENT 0x20 /* enable fsw IP netagent */ 67 68 /* 69 * Enabling Skywalk channel (user) networking stack fundamentally requires 70 * the presence of netif nexus attached to the interface. Native Skywalk 71 * drivers will come with a netif nexus; non-native drivers will require us 72 * to create a netif compat nexus, set through IF_ATTACH_NX_NETIF_COMPAT. 73 * 74 * The flowswitch nexus creation depends on the presence of netif nexus on 75 * the interface. Plumbing the flowswitch atop netif is required to connect 76 * the interface to the host (kernel) networking stack; this is the default 77 * behavior unless IF_ATTACH_NX_FLOWSWITCH is not set. 78 * 79 * To further allow for channel (user) networking stack, the netagent gets 80 * enabled on the flowswitch by default, unless opted out by clearing the 81 * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT flag. Note that 82 * IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT cannot be changed after boot, and so has 83 * to be set by the if_attach_nx bootarg. 84 */ 85 #define SKYWALK_NETWORKING_ENABLED \ 86 (IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH | \ 87 IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT) 88 89 /* 90 * To partially enable Skywalk to only let the host (kernel) stack to work, 91 * we set both IF_ATTACH_NX_NETIF_COMPAT and IF_ATTACH_NX_FLOWSWITCH flags, 92 * leaving IF_ATTACH_NX_FSW_*_NETAGENT unset. This enables netif and 93 * flowswitch flowswitch nexus on all eligible interfaces, except that channel 94 * (user) networking stack will be disabled. 95 */ 96 #define SKYWALK_NETWORKING_BSD_ONLY \ 97 (IF_ATTACH_NX_NETIF_COMPAT | IF_ATTACH_NX_FLOWSWITCH) 98 99 /* 100 * macOS default configuration for enabling support for interpose filters, 101 * custom IP, custom ether type providers and user networking stack. 102 */ 103 #define SKYWALK_NETWORKING_MAC_OS \ 104 (SKYWALK_NETWORKING_BSD_ONLY | IF_ATTACH_NX_FSW_IP_NETAGENT | \ 105 IF_ATTACH_NX_NETIF_NETAGENT | IF_ATTACH_NX_FSW_TRANSPORT_NETAGENT) 106 107 /* 108 * Disabling Skywalk networking is done by removing IF_ATTACH_NX_NETIF_COMPAT 109 * and IF_ATTACH_NX_FSW_*_NETAGENT flags, and leaving IF_ATTACH_NX_FLOWSWITCH 110 * set. This disables the compat netif instances and netagent, while still 111 * allowing flowswitch nexus creation for native Skywalk drivers, since 112 * otherwise they cease to function due to missing interaction with the host 113 * (kernel) stack. 114 */ 115 #define SKYWALK_NETWORKING_DISABLED IF_ATTACH_NX_FLOWSWITCH 116 117 #if !XNU_TARGET_OS_OSX 118 #ifdef __LP64__ 119 #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_ENABLED 120 #else /* !__LP64__ */ 121 #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_DISABLED 122 #endif /* !__LP64__ */ 123 #else /* XNU_TARGET_OS_OSX */ 124 #define IF_ATTACH_NX_DEFAULT SKYWALK_NETWORKING_MAC_OS 125 #endif /* XNU_TARGET_OS_OSX */ 126 127 #define SK_VERBOSE_SYSCTL "kern.skywalk.verbose" 128 /* 129 * Verbose flags. 130 * 131 * When adding new ones, consider the existing scheme based on the areas; 132 * try to "fill in the holes" first before extending the range. 133 */ 134 #define SK_VERB_FLAGS_TABLE(X) \ 135 X(SK_VERB_DEFAULT, 0) /* 0x0000000000000001 */ \ 136 X(SK_VERB_DUMP, 1) /* 0x0000000000000002 */ \ 137 X(SK_VERB_LOCKS, 2) /* 0x0000000000000004 */ \ 138 X(SK_VERB_REFCNT, 3) /* 0x0000000000000008 */ \ 139 X(SK_VERB_MEM, 4) /* 0x0000000000000010 */ \ 140 X(SK_VERB_MEM_ARENA, 5) /* 0x0000000000000020 */ \ 141 X(SK_VERB_MEM_CACHE, 6) /* 0x0000000000000040 */ \ 142 X(SK_VERB_MEM_REGION, 7) /* 0x0000000000000080 */ \ 143 X(SK_VERB_EVENTS, 8) /* 0x0000000000000100 */ \ 144 X(SK_VERB_SYNC, 9) /* 0x0000000000000200 */ \ 145 X(SK_VERB_NOTIFY, 10) /* 0x0000000000000400 */ \ 146 X(SK_VERB_INTR, 11) /* 0x0000000000000800 */ \ 147 X(SK_VERB_MONITOR, 12) /* 0x0000000000001000 */ \ 148 X(SK_VERB_DEV, 13) /* 0x0000000000002000 */ \ 149 X(SK_VERB_HOST, 14) /* 0x0000000000004000 */ \ 150 X(SK_VERB_USER, 15) /* 0x0000000000008000 */ \ 151 X(SK_VERB_RX, 16) /* 0x0000000000010000 */ \ 152 X(SK_VERB_TX, 17) /* 0x0000000000020000 */ \ 153 X(SK_VERB_LOOKUP, 18) /* 0x0000000000040000 */ \ 154 X(SK_VERB_RING, 19) /* 0x0000000000080000 */ \ 155 X(SK_VERB_NETIF, 20) /* 0x0000000000100000 */ \ 156 X(SK_VERB_NETIF_MIT, 21) /* 0x0000000000200000 */ \ 157 X(SK_VERB_IOSK, 22) /* 0x0000000000400000 */ \ 158 X(SK_VERB_CHANNEL, 23) /* 0x0000000000800000 */ \ 159 X(SK_VERB_AQM, 25) /* 0x0000000002000000 */ \ 160 X(SK_VERB_FSW, 24) /* 0x0000000001000000 */ \ 161 X(SK_VERB_FSW_DP, 26) /* 0x0000000004000000 */ \ 162 X(SK_VERB_LLINK, 27) /* 0x0000000008000000 */ \ 163 X(SK_VERB_FLOW, 28) /* 0x0000000010000000 */ \ 164 X(SK_VERB_FLOW_CLASSIFY, 29) /* 0x0000000020000000 */ \ 165 X(SK_VERB_FLOW_TRACK, 30) /* 0x0000000040000000 */ \ 166 X(SK_VERB_FLOW_ADVISORY, 31) /* 0x0000000080000000 */ \ 167 X(SK_VERB_FLOW_ROUTE, 32) /* 0x0000000100000000 */ \ 168 X(SK_VERB_FPD, 33) /* 0x0000000200000000 */ \ 169 X(__SK_VERB_34__, 34) /* 0x0000000400000000 */ \ 170 X(__SK_VERF_35__, 35) /* 0x0000000800000000 */ \ 171 X(SK_VERB_USER_PIPE, 36) /* 0x0000001000000000 */ \ 172 X(SK_VERB_NA, 37) /* 0x0000002000000000 */ \ 173 X(SK_VERB_KERNEL_PIPE, 38) /* 0x0000004000000000 */ \ 174 X(SK_VERB_NS_PROTO, 39) /* 0x0000008000000000 */ \ 175 X(SK_VERB_NS_TCP, 40) /* 0x0000010000000000 */ \ 176 X(SK_VERB_NS_UDP, 41) /* 0x0000020000000000 */ \ 177 X(SK_VERB_NS_IPV4, 42) /* 0x0000040000000000 */ \ 178 X(SK_VERB_NS_IPV6, 43) /* 0x0000080000000000 */ \ 179 X(SK_VERB_COPY, 44) /* 0x0000100000000000 */ \ 180 X(SK_VERB_COPY_MBUF, 45) /* 0x0000200000000000 */ \ 181 X(SK_VERB_MOVE, 46) /* 0x0000400000000000 */ \ 182 X(SK_VERB_MOVE_MBUF, 47) /* 0x0000800000000000 */ \ 183 X(SK_VERB_IP_FRAG, 48) /* 0x0001000000000000 */ \ 184 X(SK_VERB_ERROR_INJECT, 49) /* 0x0002000000000000 */ \ 185 X(SK_VERB_QOS, 50) /* 0x0004000000000000 */ \ 186 X(SK_VERB_NXPORT, 51) /* 0x0008000000000000 */ \ 187 X(SK_VERB_FILTER, 52) /* 0x0010000000000000 */ \ 188 X(SK_VERB_VP, 53) /* 0x0020000000000000 */ \ 189 X(SK_VERB_NETIF_POLL, 54) /* 0x0040000000000000 */ \ 190 X(SK_VERB_DROP, 55) /* 0x0080000000000000 */ \ 191 X(__SK_VERB_56__, 56) /* 0x0100000000000000 */ \ 192 X(__SK_VERB_57__, 57) /* 0x0200000000000000 */ \ 193 X(__SK_VERB_58__, 58) /* 0x0400000000000000 */ \ 194 X(__SK_VERB_59__, 59) /* 0x0800000000000000 */ \ 195 X(__SK_VERB_60__, 60) /* 0x1000000000000000 */ \ 196 X(__SK_VERB_61__, 61) /* 0x2000000000000000 */ \ 197 X(SK_VERB_PRIV, 62) /* 0x4000000000000000 */ \ 198 X(SK_VERB_ERROR, 63) /* 0x8000000000000000 */ 199 200 #define EXPAND_TO_STRING(name, bitshift) #name, 201 #define EXPAND_TO_ENUMERATION(name, bitshift) name = (1ULL << bitshift), 202 203 static const char *sk_verb_flags_string[] = { 204 SK_VERB_FLAGS_TABLE(EXPAND_TO_STRING) 205 }; 206 207 enum SK_VERB_FLAGS { 208 SK_VERB_FLAGS_TABLE(EXPAND_TO_ENUMERATION) 209 }; 210 211 #define SK_VERB_FLAGS_STRINGS_MAX \ 212 (sizeof (sk_verb_flags_string) / sizeof (sk_verb_flags_string[0])) 213 214 #undef EXPAND_TO_STRING 215 #undef EXPAND_TO_ENUMERATION 216 217 #ifdef KERNEL 218 #include <stdint.h> 219 #include <sys/types.h> 220 #include <sys/time.h> 221 #include <mach/vm_types.h> 222 #include <mach/vm_param.h> 223 #include <kern/cpu_number.h> 224 #include <pexpert/pexpert.h> 225 226 #if (DEVELOPMENT || DEBUG) 227 #define SK_KVA(p) ((uint64_t)(p)) 228 #define SK_LOG 1 229 #else 230 #define SK_KVA(p) ((uint64_t)VM_KERNEL_ADDRPERM(p)) 231 #define SK_LOG 0 232 #endif /* !DEVELOPMENT && !DEBUG */ 233 234 #if SK_LOG 235 #define SK_LOG_VAR(x) x 236 #else 237 #define SK_LOG_VAR(x) 238 #endif 239 240 #define SK_INLINE_ATTRIBUTE __attribute__((always_inline)) 241 #define SK_NO_INLINE_ATTRIBUTE __attribute__((noinline)) 242 #define SK_LOG_ATTRIBUTE __attribute__((noinline, cold, not_tail_called)) 243 244 #if SK_LOG 245 /* 246 * Because the compiler doesn't know about the %b format specifier, 247 * most warnings for _SK_D are disabled by pragma. 248 * 249 * XXX [email protected]: This means the compiler will not warn us about 250 * invalid parameters passed to kprintf(), so make sure to scrutinize 251 * any changes made to code using any logging macros defined below. 252 */ 253 254 extern uint64_t sk_verbose; 255 #define _SK_D(_flag, _fmt, ...) do { \ 256 if (__improbable(((_flag) && (sk_verbose & (_flag)) == (_flag)) || \ 257 (_flag) == SK_VERB_ERROR)) { \ 258 _Pragma("clang diagnostic push") \ 259 _Pragma("clang diagnostic ignored \"-Wformat-invalid-specifier\"") \ 260 _Pragma("clang diagnostic ignored \"-Wformat-extra-args\"") \ 261 _Pragma("clang diagnostic ignored \"-Wformat\"") \ 262 kprintf("SK[%u]: %-30s " _fmt "\n", \ 263 cpu_number(), __FUNCTION__, ##__VA_ARGS__); \ 264 _Pragma("clang diagnostic pop") \ 265 } \ 266 } while (0) 267 268 #define SK_DF(_flag, _fmt, ...) _SK_D((uint64_t)_flag, _fmt, ##__VA_ARGS__) 269 #define SK_D(_fmt, ...) SK_DF(SK_VERB_DEFAULT, _fmt, ##__VA_ARGS__) 270 #define SK_ERR(_fmt, ...) SK_DF(SK_VERB_ERROR, _fmt, ##__VA_ARGS__) 271 #define SK_DSC(_p, _fmt, ...) SK_ERR("%s(%d): " _fmt, \ 272 sk_proc_name_address(_p), sk_proc_pid(_p), ##__VA_ARGS__) 273 274 /* rate limited, lps indicates how many per second */ 275 #define _SK_RD(_flag, _lps, _fmt, ...) do { \ 276 static int __t0, __now, __cnt; \ 277 __now = (int)_net_uptime; \ 278 if (__t0 != __now) { \ 279 __t0 = __now; \ 280 __cnt = 0; \ 281 } \ 282 if (__cnt++ < (_lps)) \ 283 SK_DF(_flag, _fmt, ##__VA_ARGS__); \ 284 } while (0) 285 286 #define SK_RDF(_flag, _lps, _fmt, ...) \ 287 _SK_RD(_flag, _lps, _fmt, ##__VA_ARGS__) 288 #define SK_RD(_lps, _fmt, ...) \ 289 SK_RDF(SK_VERB_DEFAULT, _lps, _fmt, ##__VA_ARGS__) 290 #define SK_RDERR(_lps, _fmt, ...) \ 291 SK_RDF(SK_VERB_ERROR, _lps, _fmt, ##__VA_ARGS__) 292 #else /* !SK_LOG */ 293 #define SK_DF(_flag, _fmt, ...) do { ((void)0); } while (0) 294 #define SK_D(_fmt, ...) do { ((void)0); } while (0) 295 #define SK_ERR(_fmt, ...) do { ((void)0); } while (0) 296 #define SK_DSC(_p, _fmt, ...) do { ((void)0); } while (0) 297 #define SK_RDF(_flag, _lps, _fmt, ...) do { ((void)0); } while (0) 298 #define SK_RD(_lps, _fmt, ...) do { ((void)0); } while (0) 299 #define SK_RDERR(_lps, _fmt, ...) do { ((void)0); } while (0) 300 #endif /* ! SK_LOG */ 301 302 #ifdef BSD_KERNEL_PRIVATE 303 #include <skywalk/core/skywalk_var.h> 304 #include <skywalk/lib/cuckoo_hashtable.h> 305 #include <skywalk/mem/skmem_var.h> 306 #include <skywalk/channel/os_channel_event.h> 307 #include <skywalk/channel/channel_var.h> 308 #include <skywalk/nexus/nexus_var.h> 309 #include <skywalk/packet/pbufpool_var.h> 310 #include <skywalk/packet/packet_var.h> 311 #endif /* BSD_KERNEL_PRIVATE */ 312 #endif /* KERNEL */ 313 #if !defined(KERNEL) || defined(BSD_KERNEL_PRIVATE) 314 #include <skywalk/skywalk_common.h> 315 #include <skywalk/os_nexus_private.h> 316 #include <skywalk/os_channel_private.h> 317 #include <skywalk/os_packet_private.h> 318 #include <skywalk/os_stats_private.h> 319 #endif /* !KERNEL || BSD_KERNEL_PRIVATE */ 320 #endif /* PRIVATE || XNU_KERNEL_PRIVATE */ 321 #endif /* _SKYWALK_OS_SKYWALK_PRIVATE_H */ 322