1 /* 2 * Copyright (c) 2015-2021 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 29 #ifndef _SKYWALK_OS_CHANNEL_H_ 30 #define _SKYWALK_OS_CHANNEL_H_ 31 32 #ifdef PRIVATE 33 34 #include <stdint.h> 35 #include <sys/types.h> 36 #include <sys/cdefs.h> 37 #include <uuid/uuid.h> 38 #include <mach/vm_types.h> 39 #include <skywalk/os_nexus.h> 40 #include <skywalk/os_packet.h> 41 #ifndef KERNEL 42 #include <skywalk/os_channel_event.h> 43 #include <net/if_var.h> 44 #endif /* !KERNEL */ 45 46 /* 47 * Indicates that channel supports "CHANNEL_ATTR_NUM_BUFFERS" attribute. 48 * used by Libnetcore. 49 */ 50 #define OS_CHANNEL_HAS_NUM_BUFFERS_ATTR 1 51 52 /* Flow advisory table index */ 53 typedef uint32_t flowadv_idx_t; 54 #define FLOWADV_IDX_NONE ((flowadv_idx_t)-1) 55 56 /* 57 * Channel ring direction. 58 */ 59 typedef enum { 60 CHANNEL_DIR_TX_RX, /* default: TX and RX ring(s) */ 61 CHANNEL_DIR_TX, /* (monitor) only TX ring(s) */ 62 CHANNEL_DIR_RX /* (monitor) only RX ring(s) */ 63 } ring_dir_t; 64 65 /* 66 * Channel ring ID. 67 */ 68 typedef uint32_t ring_id_t; 69 #define CHANNEL_RING_ID_ANY ((ring_id_t)-1) 70 71 typedef enum { 72 CHANNEL_FIRST_TX_RING, 73 CHANNEL_LAST_TX_RING, 74 CHANNEL_FIRST_RX_RING, 75 CHANNEL_LAST_RX_RING 76 } ring_id_type_t; 77 78 /* Sync mode values */ 79 typedef enum { 80 CHANNEL_SYNC_TX, /* synchronize TX ring(s) */ 81 CHANNEL_SYNC_RX, /* synchronize RX ring(s) */ 82 #if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE) 83 CHANNEL_SYNC_UPP /* synchronize packet pool rings only */ 84 #endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */ 85 } sync_mode_t; 86 87 /* Sync flags */ 88 typedef uint32_t sync_flags_t; 89 #if defined(LIBSYSCALL_INTERFACE) || defined(BSD_KERNEL_PRIVATE) 90 #define CHANNEL_SYNCF_ALLOC 0x1 /* synchronize alloc ring */ 91 #define CHANNEL_SYNCF_FREE 0x2 /* synchronize free ring */ 92 #define CHANNEL_SYNCF_PURGE 0x4 /* purge user packet pool */ 93 #define CHANNEL_SYNCF_ALLOC_BUF 0x8 /* synchronize buflet alloc ring */ 94 #endif /* LIBSYSCALL_INTERFACE || BSD_KERNEL_PRIVATE */ 95 96 /* 97 * Opaque handles. 98 */ 99 struct channel; 100 struct channel_ring_desc; 101 struct __slot_desc; 102 struct channel_attr; 103 104 typedef struct channel *channel_t; 105 typedef struct channel_ring_desc *channel_ring_t; 106 typedef struct __slot_desc *channel_slot_t; 107 typedef struct channel_attr *channel_attr_t; 108 109 /* 110 * Channel monitor types. 111 */ 112 typedef enum { 113 CHANNEL_MONITOR_OFF, /* default */ 114 CHANNEL_MONITOR_NO_COPY, /* zero-copy (delayed) mode */ 115 CHANNEL_MONITOR_COPY /* copy (immediate) mode */ 116 } channel_monitor_type_t; 117 118 /* 119 * Channel threshold unit types. 120 */ 121 typedef enum { 122 CHANNEL_THRESHOLD_UNIT_SLOTS, /* unit in slots (default) */ 123 CHANNEL_THRESHOLD_UNIT_BYTES, /* unit in bytes */ 124 } channel_threshold_unit_t; 125 126 /* 127 * Channel attribute types gettable/settable via os_channel_attr_{get,set}. 128 * 129 * g: retrievable at any time 130 * s: settable at any time 131 * S: settable once, only at creation time 132 */ 133 typedef enum { 134 CHANNEL_ATTR_TX_RINGS, /* (g) # of transmit rings */ 135 CHANNEL_ATTR_RX_RINGS, /* (g) # of receive rings */ 136 CHANNEL_ATTR_TX_SLOTS, /* (g) # of slots per transmit ring */ 137 CHANNEL_ATTR_RX_SLOTS, /* (g) # of slots per receive ring */ 138 CHANNEL_ATTR_SLOT_BUF_SIZE, /* (g) buffer per slot (bytes) */ 139 CHANNEL_ATTR_SLOT_META_SIZE, /* (g) metadata per slot (bytes) */ 140 CHANNEL_ATTR_EXCLUSIVE, /* (g/s) bool: exclusive open */ 141 CHANNEL_ATTR_NO_AUTO_SYNC, /* (g/s) bool: will do explicit sync */ 142 CHANNEL_ATTR_MONITOR, /* (g/s) see channel_monitor_type_t */ 143 CHANNEL_ATTR_TX_LOWAT_UNIT, /* (g/s) see channel_threshold_unit_t */ 144 CHANNEL_ATTR_TX_LOWAT_VALUE, /* (g/s) transmit low-watermark */ 145 CHANNEL_ATTR_RX_LOWAT_UNIT, /* (g/s) see channel_threshold_unit_t */ 146 CHANNEL_ATTR_RX_LOWAT_VALUE, /* (g/s) receive low-watermark */ 147 CHANNEL_ATTR_NEXUS_TYPE, /* (g) nexus type */ 148 CHANNEL_ATTR_NEXUS_EXTENSIONS, /* (g) nexus extension(s) */ 149 CHANNEL_ATTR_NEXUS_MHINTS, /* (g) nexus memory usage hints */ 150 CHANNEL_ATTR_TX_HOST_RINGS, /* (g) # of transmit host rings */ 151 CHANNEL_ATTR_RX_HOST_RINGS, /* (g) # of receive host rings */ 152 CHANNEL_ATTR_NEXUS_IFINDEX, /* (g) nexus network interface index */ 153 CHANNEL_ATTR_NEXUS_STATS_SIZE, /* (g) nexus statistics region size */ 154 CHANNEL_ATTR_NEXUS_FLOWADV_MAX, /* (g) # of flow advisory entries */ 155 CHANNEL_ATTR_NEXUS_META_TYPE, /* (g) nexus metadata type */ 156 CHANNEL_ATTR_NEXUS_META_SUBTYPE, /* (g) nexus metadata subtype */ 157 CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD, /* (g) nexus checksum offload */ 158 CHANNEL_ATTR_USER_PACKET_POOL, /* (g/s) bool: use user packet pool */ 159 CHANNEL_ATTR_NEXUS_ADV_SIZE, /* (g) nexus advisory region size */ 160 CHANNEL_ATTR_NEXUS_DEFUNCT_OK, /* (g/s) bool: allow defunct */ 161 CHANNEL_ATTR_FILTER, /* (g/s) bool: filter mode */ 162 CHANNEL_ATTR_EVENT_RING, /* (g/s) bool: enable event ring */ 163 CHANNEL_ATTR_MAX_FRAGS, /* (g) max length of buflet chain */ 164 CHANNEL_ATTR_NUM_BUFFERS, /* (g) # of buffers in user pool */ 165 CHANNEL_ATTR_LOW_LATENCY, /* (g/s) bool: low latency channel */ 166 CHANNEL_ATTR_LARGE_BUF_SIZE, /* (g) large buffer size (bytes) */ 167 } channel_attr_type_t; 168 169 /* 170 * Channel nexus metadata type. 171 */ 172 typedef enum { 173 CHANNEL_NEXUS_META_TYPE_INVALID = 0, 174 CHANNEL_NEXUS_META_TYPE_QUANTUM, /* OK for os_packet quantum APIs */ 175 CHANNEL_NEXUS_META_TYPE_PACKET, /* OK for all os_packet APIs */ 176 } channel_nexus_meta_type_t; 177 178 /* 179 * Channel nexus metadata subtype. 180 */ 181 typedef enum { 182 CHANNEL_NEXUS_META_SUBTYPE_INVALID = 0, 183 CHANNEL_NEXUS_META_SUBTYPE_PAYLOAD, 184 CHANNEL_NEXUS_META_SUBTYPE_RAW, 185 } channel_nexus_meta_subtype_t; 186 187 /* 188 * Valid values for CHANNEL_ATTR_NEXUS_CHECKSUM_OFFLOAD 189 */ 190 #define CHANNEL_NEXUS_CHECKSUM_PARTIAL 0x1 /* partial checksum */ 191 192 /* 193 * Channel statistics ID. 194 */ 195 typedef enum { 196 CHANNEL_STATS_ID_IP = 0, /* struct ip_stats */ 197 CHANNEL_STATS_ID_IP6, /* struct ip6_stats */ 198 CHANNEL_STATS_ID_TCP, /* struct tcp_stats */ 199 CHANNEL_STATS_ID_UDP, /* struct udp_stats */ 200 CHANNEL_STATS_ID_QUIC, /* struct quic_stats */ 201 } channel_stats_id_t; 202 203 /* 204 * Slot properties. Structure is aligned to allow for efficient copy. 205 * 206 * Fields except for sp_{flags,len} are immutables (I). The system will 207 * verify for correctness during os_channel_put() across the immutable 208 * fields, and will abort the process if it detects inconsistencies. 209 * This is meant to help with debugging, since it indicates bugs and/or 210 * memory corruption. 211 */ 212 typedef struct slot_prop { 213 uint16_t sp_flags; /* private flags */ 214 uint16_t sp_len; /* length for this slot */ 215 uint32_t sp_idx; /* (I) slot index */ 216 mach_vm_address_t sp_ext_ptr; /* (I) pointer for indirect buffer */ 217 mach_vm_address_t sp_buf_ptr; /* (I) pointer for native buffer */ 218 mach_vm_address_t sp_mdata_ptr; /* (I) pointer for metadata */ 219 uint32_t _sp_pad[8]; /* reserved */ 220 } slot_prop_t __attribute__((aligned(sizeof(uint64_t)))); 221 222 #ifndef KERNEL 223 /* 224 * User APIs. 225 */ 226 #if !defined(_POSIX_C_SOURCE) || defined(_DARWIN_C_SOURCE) 227 __BEGIN_DECLS 228 /* 229 * Creates a Channel attribute object. 230 * 231 * This must be paired with a os_channel_attr_destroy() on the handle. 232 */ 233 extern channel_attr_t os_channel_attr_create(void); 234 235 /* 236 * Clones a Channel attribute object. If source attribute is NULL 237 * it behaves just like os_channel_attr_create(); 238 * 239 * This must be paired with a os_channel_attr_destroy() on the handle. 240 */ 241 extern channel_attr_t os_channel_attr_clone(const channel_attr_t); 242 243 /* 244 * Sets a value for a given attribute type on a Channel attribute object. 245 */ 246 extern int os_channel_attr_set(const channel_attr_t attr, 247 const channel_attr_type_t type, const uint64_t value); 248 249 /* 250 * Sets a key blob on a Channel attribute object. Existing key blob 251 * information in the attribute object will be removed, if any, and 252 * replaced with the new key blob. Specifying 0 for key_len will 253 * clear the key stored in the attribute object. The maximum key 254 * length is specified by NEXUS_MAX_KEY_LEN. 255 */ 256 extern int os_channel_attr_set_key(const channel_attr_t attr, 257 const void *key, const uint32_t key_len); 258 259 /* 260 * Gets a value for a given attribute type on a Channel attribute object. 261 */ 262 extern int os_channel_attr_get(const channel_attr_t attr, 263 const channel_attr_type_t type, uint64_t *value); 264 265 /* 266 * Gets a key blob on a Channel attribute object. If key is NULL, 267 * returns the length of the key blob with key_len, so caller knows 268 * how much to allocate space for key blob. 269 */ 270 extern int os_channel_attr_get_key(const channel_attr_t attr, 271 void *key, uint32_t *key_len); 272 273 /* 274 * Destroys a Channel attribute object, along with all resources 275 * associated with it (e.g. key blob). 276 */ 277 extern void os_channel_attr_destroy(const channel_attr_t attr); 278 279 /* 280 * Opens a Channel to a Nexus provider instance. Upon success, maps memory 281 * region and allocates resources. 282 * 283 * This must be paired with a os_channel_destroy() on the handle, in order to 284 * unmap the memory region and free resources. 285 */ 286 extern channel_t os_channel_create(const uuid_t uuid, const nexus_port_t port); 287 288 /* 289 * Extended version of os_channel_create(). 290 */ 291 extern channel_t os_channel_create_extended(const uuid_t uuid, 292 const nexus_port_t port, const ring_dir_t dir, const ring_id_t rid, 293 const channel_attr_t attr); 294 295 /* 296 * Retrieves the file descriptor associated with the Channel. 297 */ 298 extern int os_channel_get_fd(const channel_t channel); 299 300 /* 301 * Retrieves current channel attributes into the channel_attr_t handle. 302 */ 303 extern int os_channel_read_attr(const channel_t channel, channel_attr_t attr); 304 305 /* 306 * Updates channel attributes based on those referred to by the channel_attr_t 307 * handle. See comments above on channel_attr_type_t; this routine will only 308 * update attributes that are marked with 's' but not 'S'. 309 */ 310 extern int os_channel_write_attr(const channel_t channel, channel_attr_t attr); 311 312 /* 313 * Retrieves channel's associated nexus type into *nexus_type, and the 314 * provider-specific extension attribute into *ext. 315 */ 316 extern int os_channel_read_nexus_extension_info(const channel_t channel, 317 nexus_type_t *nexus_type, uint64_t *ext); 318 319 /* 320 * Non-blocking synchronization. Channel handle may also be used 321 * with kqueue(2), select(2) or poll(2) through the file descriptor. 322 */ 323 extern int os_channel_sync(const channel_t channel, const sync_mode_t mode); 324 325 /* 326 * Destroys a Channel. 327 */ 328 extern void os_channel_destroy(const channel_t channel); 329 330 /* 331 * Checks if a channel is defunct. Returns non-zero if defunct. 332 */ 333 extern int os_channel_is_defunct(const channel_t channel); 334 335 /* 336 * Data Movement APIs. 337 * 338 * Obtain channel_ring_t handle via os_channel_{tx,rx}_ring(). You will 339 * need to specify the ring_id_t which identifies the ring — this is true 340 * even for a single TX/RX ring case. The Nexus provider can communicate 341 * to the client the ID of the TX and RX ring that should be used to 342 * communicate to it, through a contract between the two. For instance, 343 * it can tell the client to use first TX ring and first RX ring, etc. 344 * through some side-channel. It should not assume 0 or any other number 345 * as ID, however, as the in-kernel Nexus object is the authoritative source 346 * of truth. This is where the os_channel_ring_id() call comes into the 347 * picture, as it will return the first and last usable TX and RX ring IDs 348 * for the Channel opened to that Nexus object. 349 * 350 * Once the TX or RX ring handle is obtained above, the client can ask for 351 * the first usable slot in the ring through os_channel_get_next_slot() 352 * passing NULL for the 'slot' parameter. This returns a channel_slot_t 353 * handle that represents the slot, along with the properties of that slot 354 * described by the slot_prop_t structure. If no slots are available, this 355 * call returns a NULL handle. It’s important to note that this 356 * call does NOT advance the ring’s current slot pointer; calling this 357 * multiple times in succession will yield the same result. 358 * 359 * The client proceeds to use the slot by examining the returned 360 * slot_prop_t fields including the pointer to the internal buffer 361 * associated with that slot. Once the client is finished, it updates 362 * the relevant slot_prop_t fields (e.g. length) and calls 363 * os_channel_set_slot_properties() to apply them to the slot. 364 * 365 * To get the next slot, the client provides the non-NULL slot value obtained 366 * from the previous call to os_channel_get_next_slot() as the 'slot' parameter 367 * in its next invocation of that function. 368 * 369 * To advance the ring’s current pointer, the client invokes 370 * os_channel_advance_slot() specifying the slot to advance past. If the slot 371 * is invalid, this function returns a non-zero value. 372 * 373 * Once the client is ready to commit, call os_channel_sync() in 374 * either/all directions. 375 */ 376 extern ring_id_t os_channel_ring_id(const channel_t channel, 377 const ring_id_type_t type); 378 extern channel_ring_t os_channel_tx_ring(const channel_t channel, 379 const ring_id_t rid); 380 extern channel_ring_t os_channel_rx_ring(const channel_t channel, 381 const ring_id_t rid); 382 extern int os_channel_pending(const channel_ring_t ring); 383 384 /* 385 * This returns a nexus-specific timestamp in nanoseconds taken at the 386 * lasttime os_channel_sync() or its equivalent implicit kevent sync 387 * was called 388 */ 389 extern uint64_t os_channel_ring_sync_time(const channel_ring_t ring); 390 391 /* 392 * This returns a nexus-specific timestamp in nanoseconds to indicate 393 * the time of last activity on the opposite end of the ring. 394 * This is only updated when sync or kevent equivalent is called. 395 */ 396 extern uint64_t os_channel_ring_notify_time(const channel_ring_t ring); 397 398 /* 399 * For TX ring os_channel_available_slot_count() returns the minimum number 400 * of slots available availble for TX, and it is possible that 401 * os_channel_get_next_slot() will return more slots than the what was 402 * returned by an earlier call to os_channel_available_slot_count() 403 */ 404 extern uint32_t os_channel_available_slot_count(const channel_ring_t ring); 405 extern channel_slot_t os_channel_get_next_slot(const channel_ring_t ring, 406 const channel_slot_t slot, slot_prop_t *prop); 407 extern int os_channel_advance_slot(channel_ring_t ring, 408 const channel_slot_t slot); 409 extern void os_channel_set_slot_properties(const channel_ring_t ring, 410 const channel_slot_t slot, const slot_prop_t *prop); 411 412 /* 413 * Return the packet handle associated with a given slot of a ring. 414 */ 415 extern packet_t os_channel_slot_get_packet(const channel_ring_t ring, 416 const channel_slot_t slot); 417 418 /* 419 * Each nexus that the channel is connected to determines whether or 420 * not there is a shareable statistics region identified by one of 421 * the channel_stats_id_t values. This routine returns a pointer to 422 * such a region upon success, or NULL if not supported by the nexus. 423 */ 424 extern void *os_channel_get_stats_region(const channel_t channel, 425 const channel_stats_id_t id); 426 427 /* 428 * Each nexus that the channel is connected to determines whether or 429 * not there is a nexus-wide advisory region. This routine returns 430 * a pointer to such a region upon success, or NULL if not supported 431 * by the nexus. 432 */ 433 extern void *os_channel_get_advisory_region(const channel_t channel); 434 435 /* 436 * Each nexus that supports flow admission control may be queried to 437 * advise whether or not the channel is willing to admit more packets 438 * for a given flow. A return value of 0 indicates that the packets 439 * for the flow are admissible. If ENOBUFS is returned, the flow is 440 * currently suspended, and further attempts to send more packets on 441 * the ring may result in drops. Any other error values indicate 442 * that either the nexus doesn't support admission control, or the 443 * arguments aren't valid. 444 */ 445 extern int os_channel_flow_admissible(const channel_ring_t ring, 446 uuid_t flow_id, const flowadv_idx_t flow_index); 447 448 /* 449 * Allocate a packet from the channel's packet pool. 450 * Returns 0 on success with the packet handle in packet arg. 451 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be 452 * serialized and should not be called from the different thread context. 453 */ 454 extern int 455 os_channel_packet_alloc(const channel_t chd, packet_t *packet); 456 457 /* 458 * Free a packet allocated from the channel's packet pool. 459 * Returns 0 on success 460 * Note: os_channel_packet_alloc() & os_channel_packet_free() should be 461 * serialized and should not be called from the different thread context. 462 */ 463 extern int 464 os_channel_packet_free(const channel_t chd, packet_t packet); 465 466 /* 467 * Attach the given packet to a channel slot 468 */ 469 extern int 470 os_channel_slot_attach_packet(const channel_ring_t chrd, 471 const channel_slot_t slot, packet_t packet); 472 473 /* 474 * Detach a given packet from a channel slot 475 */ 476 extern int 477 os_channel_slot_detach_packet(const channel_ring_t chrd, 478 const channel_slot_t slot, packet_t packet); 479 480 /* 481 * purge packets from the channel's packet pool. 482 * This API should be called at regular intervals by application to purge 483 * unused packets from the channel's packet pool. Recommended interval is 484 * 11 seconds. 485 * Returns 0 on success. 486 * Note: This call should be serialized with os_channel_packet_alloc() & 487 * os_channel_packet_free() and should not be called from different 488 * thread context. 489 */ 490 extern int 491 os_channel_packet_pool_purge(const channel_t chd); 492 493 /* 494 * Retrieve handle to the next available event(s) on the channel. 495 * os_event_get_next_event() can then called on the event handle to 496 * retrieve the individual events from the handle. 497 * Returns 0 on success, ENXIO if the channel is defunct. 498 */ 499 extern int 500 os_channel_get_next_event_handle(const channel_t chd, 501 os_channel_event_handle_t *ehandle, os_channel_event_type_t *etype, 502 uint32_t *nevents); 503 504 /* 505 * Free an event retrieved from the channel. 506 * Returns 0 on success, ENXIO if the channel is defunct. 507 */ 508 extern int 509 os_channel_event_free(const channel_t chd, os_channel_event_handle_t ehandle); 510 511 /* 512 * API to retrieve the latest interface advisory report on the channel. 513 * Returns 0 on succcess. If the return value is EAGAIN, caller can attempt 514 * to retrieve the information again. 515 */ 516 extern int 517 os_channel_get_interface_advisory(const channel_t chd, 518 struct ifnet_interface_advisory *advisory); 519 520 /* 521 * API to configure interface advisory report on the channel. 522 * Returns 0 on succcess. 523 */ 524 extern int 525 os_channel_configure_interface_advisory(const channel_t chd, boolean_t enable); 526 527 extern int 528 os_channel_buflet_alloc(const channel_t chd, buflet_t *bft); 529 530 extern int 531 os_channel_buflet_free(const channel_t chd, buflet_t ubft); 532 __END_DECLS 533 #endif /* (!_POSIX_C_SOURCE || _DARWIN_C_SOURCE) */ 534 #else /* KERNEL */ 535 /* 536 * Kernel APIs. 537 */ 538 539 /* 540 * Opaque handles. 541 */ 542 struct kern_channel; 543 struct __kern_channel_ring; 544 545 typedef struct kern_channel *kern_channel_t; 546 typedef struct __kern_channel_ring *kern_channel_ring_t; 547 typedef struct __slot_desc *kern_channel_slot_t; 548 549 /* 550 * Slot properties (deprecated). 551 */ 552 struct kern_slot_prop { 553 uint32_t _sp_pad[16]; /* reserved */ 554 } __attribute__((aligned(sizeof(uint64_t)))); 555 556 /* 557 * @struct kern_channel_ring_stat_increment 558 * @abstract Structure used to increment the per ring statistic counters. 559 * @field kcrsi_slots_transferred number of slots transferred 560 * @filed kcrsi_bytes_transferred number of bytes transferred 561 */ 562 struct kern_channel_ring_stat_increment { 563 uint32_t kcrsi_slots_transferred; 564 uint32_t kcrsi_bytes_transferred; 565 }; 566 567 /* 568 * Data Movement APIs. 569 * 570 * See block comment above for userland data movement APIs for general 571 * concepts. The main differences here are the kern_channel_notify() 572 * and kern_channel_reclaim() calls that aren't available for userland. 573 * These calls are typically invoked within the TX and RX sync callbacks 574 * implemented by the nexus provider. 575 * 576 * For TX sync, kern_channel_reclaim() is normally called after the 577 * provider has finished reclaiming slots that have been "transmitted". 578 * In this case, this call is simply a way to indicate to the system 579 * that such condition has happened. 580 * 581 * For RX sync, kern_channel_reclaim() must be called at the beginning 582 * of the callback in order to reclaim user-released slots, and to 583 * ensure that subsequent calls to kern_channel_available_slot_count() 584 * or kern_channel_get_next_slot() operates on the most recent state. 585 * 586 * The kern_channel_notify() is used to post notifications to indicate 587 * slot availability; this may result in the kernel event subsystem 588 * posting readable and writable events. 589 */ 590 __BEGIN_DECLS 591 extern uint32_t kern_channel_notify(const kern_channel_ring_t, uint32_t flags); 592 extern uint32_t kern_channel_available_slot_count( 593 const kern_channel_ring_t ring); 594 /* 595 * NOTE: kern_channel_set_slot_properties(), kern_channel_get_next_slot(), 596 * kern_channel_reclaim() and kern_channel_advance_slot() require that the 597 * caller invokes them from within the sync callback context; they will 598 * assert otherwise. 599 */ 600 extern void kern_channel_set_slot_properties(const kern_channel_ring_t, 601 const kern_channel_slot_t slot, const struct kern_slot_prop *prop); 602 extern kern_channel_slot_t kern_channel_get_next_slot( 603 const kern_channel_ring_t kring, const kern_channel_slot_t slot, 604 struct kern_slot_prop *slot_prop); 605 extern uint32_t kern_channel_reclaim(const kern_channel_ring_t); 606 extern void kern_channel_advance_slot(const kern_channel_ring_t kring, 607 kern_channel_slot_t slot); 608 609 /* 610 * Packet. 611 */ 612 extern kern_packet_t kern_channel_slot_get_packet( 613 const kern_channel_ring_t ring, const kern_channel_slot_t slot); 614 615 /* 616 * NOTE: kern_channel_slot_attach_packet(), kern_channel_slot_detach_packet() 617 * and kern_channel_ring_get_container() require that the caller invokes them 618 * from within the sync callback context; they will assert otherwise. 619 */ 620 extern errno_t kern_channel_slot_attach_packet(const kern_channel_ring_t ring, 621 const kern_channel_slot_t slot, kern_packet_t packet); 622 extern errno_t kern_channel_slot_detach_packet(const kern_channel_ring_t ring, 623 const kern_channel_slot_t slot, kern_packet_t packet); 624 extern errno_t kern_channel_ring_get_container(const kern_channel_ring_t ring, 625 kern_packet_t **array, uint32_t *count); 626 extern errno_t kern_channel_tx_refill(const kern_channel_ring_t ring, 627 uint32_t pkt_limit, uint32_t byte_limit, boolean_t tx_doorbell_ctxt, 628 boolean_t *pkts_pending); 629 extern errno_t kern_channel_get_service_class(const kern_channel_ring_t ring, 630 kern_packet_svc_class_t *svc); 631 extern errno_t kern_netif_queue_get_service_class(kern_netif_queue_t, 632 kern_packet_svc_class_t *); 633 634 /* 635 * Misc. 636 */ 637 extern void *kern_channel_get_context(const kern_channel_t channel); 638 extern void *kern_channel_ring_get_context(const kern_channel_ring_t ring); 639 extern void *kern_channel_slot_get_context(const kern_channel_ring_t ring, 640 const kern_channel_slot_t slot); 641 642 /* 643 * NOTE: kern_channel_increment_ring_{net}_stats() requires 644 * that the caller invokes it from within the sync callback context; 645 * it will assert otherwise. 646 */ 647 extern void kern_channel_increment_ring_stats(kern_channel_ring_t ring, 648 struct kern_channel_ring_stat_increment *stats); 649 extern void kern_channel_increment_ring_net_stats(kern_channel_ring_t ring, 650 ifnet_t, struct kern_channel_ring_stat_increment *stats); 651 652 #ifdef BSD_KERNEL_PRIVATE 653 /* forward declare */ 654 struct flowadv_fcentry; 655 656 /* Flow advisory token */ 657 typedef uint32_t flowadv_token_t; 658 659 /* 660 * Private, unexported KPIs. 661 */ 662 __private_extern__ errno_t kern_channel_slot_attach_packet_byidx( 663 const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph); 664 __private_extern__ errno_t kern_channel_slot_detach_packet_byidx( 665 const kern_channel_ring_t kring, const uint32_t sidx, kern_packet_t ph); 666 __private_extern__ void kern_channel_flowadv_clear(struct flowadv_fcentry *); 667 __private_extern__ void kern_channel_memstatus(struct proc *, uint32_t, 668 struct kern_channel *); 669 __private_extern__ void kern_channel_defunct(struct proc *, 670 struct kern_channel *); 671 __private_extern__ errno_t kern_channel_tx_refill_canblock( 672 const kern_channel_ring_t, uint32_t, uint32_t, boolean_t, boolean_t *); 673 #endif /* BSD_KERNEL_PRIVATE */ 674 __END_DECLS 675 #endif /* KERNEL */ 676 #endif /* PRIVATE */ 677 #endif /* !_SKYWALK_OS_CHANNEL_H_ */ 678