xref: /xnu-11417.121.6/bsd/net/if_fake.c (revision a1e26a70f38d1d7daa7b49b258e2f8538ad81650)

/*
 * Copyright (c) 2015-2024 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

/*
 * if_fake.c
 * - fake network interface used for testing
 * - "feth" (e.g. "feth0", "feth1") is a virtual ethernet interface that allows
 *   two instances to have their output/input paths "crossed-over" so that
 *   output on one is input on the other
 */

/*
 * Modification History:
 *
 * September 9, 2015	Dieter Siegmund ([email protected])
 * - created
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mbuf.h>
#include <sys/queue.h>
#include <sys/socket.h>
#include <sys/sockio.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/kern_event.h>
#include <sys/mcache.h>
#include <sys/syslog.h>

#include <net/bpf.h>
#include <net/ethernet.h>
#include <net/if.h>
#include <net/if_vlan_var.h>
#include <net/if_fake_var.h>
#include <net/if_arp.h>
#include <net/if_dl.h>
#include <net/if_ether.h>
#include <net/if_types.h>
#include <libkern/OSAtomic.h>

#include <net/dlil.h>

#include <net/kpi_interface.h>
#include <net/kpi_protocol.h>

#include <kern/locks.h>
#include <kern/zalloc.h>

#include <mach/mach_time.h>

#include <os/log.h>

#ifdef INET
#include <netinet/in.h>
#include <netinet/if_ether.h>
#endif

#include <net/if_media.h>
#include <net/ether_if_module.h>
#if SKYWALK
#include <skywalk/os_skywalk_private.h>
#include <skywalk/nexus/netif/nx_netif.h>
#include <skywalk/channel/channel_var.h>
#endif /* SKYWALK */

/*
 * if_fake_debug, FE_DBGF_*
 * - 'if_fake_debug' is a bitmask of FE_DBGF_* flags that can be set
 *   to enable additional logs for the corresponding fake function
 * - "sysctl net.link.fake.debug" controls the value of
 *   'if_fake_debug'
 */
static uint32_t if_fake_debug = 0;

#define FE_DBGF_LIFECYCLE               0x0001
#define FE_DBGF_INPUT                   0x0002
#define FE_DBGF_OUTPUT                  0x0004
#define FE_DBGF_CONTROL                 0x0008
#define FE_DBGF_MISC                    0x0010

/*
 * if_fake_log_level
 * - 'if_fake_log_level' ensures that by default important logs are
 *   logged regardless of if_fake_debug by comparing the log level
 *   in FAKE_LOG to if_fake_log_level
 * - use "sysctl net.link.fake.log_level" controls the value of
 *   'if_fake_log_level'
 * - the default value of 'if_fake_log_level' is LOG_NOTICE; important
 *   logs must use LOG_NOTICE to ensure they appear by default
 */
#define FAKE_DBGF_ENABLED(__flag)     ((if_fake_debug & __flag) != 0)

/*
 * FAKE_LOG
 * - macro to generate the specified log conditionally based on
 *   the specified log level and debug flags
 */
#define FAKE_LOG(__level, __dbgf, __string, ...)              \
	do {                                                            \
	        if (__level <= if_fake_log_level ||                   \
	            FAKE_DBGF_ENABLED(__dbgf)) {                      \
	                os_log(OS_LOG_DEFAULT, "%s: " __string, \
	                       __func__, ## __VA_ARGS__);       \
	        }                                                       \
	} while (0)

static boolean_t
is_power_of_two(unsigned int val)
{
	return (val & (val - 1)) == 0;
}

#define FAKE_ETHER_NAME         "feth"

SYSCTL_DECL(_net_link);
SYSCTL_NODE(_net_link, OID_AUTO, fake, CTLFLAG_RW | CTLFLAG_LOCKED, 0,
    "Fake interface");

static int if_fake_txstart = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, txstart, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_txstart, 0, "Fake interface TXSTART mode");

static int if_fake_hwcsum = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, hwcsum, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_hwcsum, 0, "Fake interface simulate hardware checksum");

static int if_fake_vlan_tagging = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, vlan_tagging, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_vlan_tagging, 0, "Fake interface VLAN tagging");

static int if_fake_nxattach = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, nxattach, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_nxattach, 0, "Fake interface auto-attach nexus");

static int if_fake_bsd_mode = 1;
SYSCTL_INT(_net_link_fake, OID_AUTO, bsd_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_bsd_mode, 0, "Fake interface attach as BSD interface");

static int if_fake_log_level = LOG_NOTICE;
SYSCTL_INT(_net_link_fake, OID_AUTO, log_level, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_log_level, 0, "Fake interface log level");

SYSCTL_INT(_net_link_fake, OID_AUTO, debug, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_debug, 0, "Fake interface debug flags");

static int if_fake_wmm_mode = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, wmm_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_wmm_mode, 0, "Fake interface in 802.11 WMM mode");

static int if_fake_multibuflet = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, multibuflet, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_multibuflet, 0, "Fake interface using multi-buflet packets");

static int if_fake_low_latency = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, low_latency, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_low_latency, 0, "Fake interface with a low latency qset");

static int if_fake_switch_combined_mode = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, switch_combined_mode,
    CTLFLAG_RW | CTLFLAG_LOCKED, &if_fake_switch_combined_mode, 0,
    "Switch a qset between combined and separate mode during dequeues");

static int if_fake_switch_mode_frequency = 10;
SYSCTL_INT(_net_link_fake, OID_AUTO, switch_mode_frequency,
    CTLFLAG_RW | CTLFLAG_LOCKED, &if_fake_switch_mode_frequency, 0,
    "The number of dequeues before we switch between the combined and separated mode");

static int if_fake_tso_support = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, tso_support, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_tso_support, 0, "Fake interface with support for TSO offload");

#define DEFAULT_EXPIRATION_THRESHOLD 500 /* usec */
static int if_fake_expiration_threshold_us = DEFAULT_EXPIRATION_THRESHOLD;
SYSCTL_INT(_net_link_fake, OID_AUTO, expiration_threshold, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_expiration_threshold_us, DEFAULT_EXPIRATION_THRESHOLD,
    "Expiration threshold (usec) for expiration testing");

static int if_fake_lro = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, lro, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_lro, 0, "Fake interface report LRO capability");

static int if_fake_separate_frame_header = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, separate_frame_header,
    CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_separate_frame_header, 0, "Put frame header in separate mbuf");

typedef enum {
	IFF_PP_MODE_GLOBAL = 0,         /* share a global pool */
	IFF_PP_MODE_PRIVATE = 1,        /* creates its own rx/tx pool */
	IFF_PP_MODE_PRIVATE_SPLIT = 2,  /* creates its own split rx & tx pool */
} iff_pktpool_mode_t;
static iff_pktpool_mode_t if_fake_pktpool_mode = IFF_PP_MODE_GLOBAL;
SYSCTL_INT(_net_link_fake, OID_AUTO, pktpool_mode, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_pktpool_mode, IFF_PP_MODE_GLOBAL,
    "Fake interface packet pool mode (0 global, 1 private, 2 private split");

#define FETH_LINK_LAYER_AGGRETATION_FACTOR_MAX 512
#define FETH_LINK_LAYER_AGGRETATION_FACTOR_DEF 96
static int if_fake_link_layer_aggregation_factor =
    FETH_LINK_LAYER_AGGRETATION_FACTOR_DEF;
static int
feth_link_layer_aggregation_factor_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_link_layer_aggregation_factor,
	    sizeof(if_fake_link_layer_aggregation_factor), &new_value,
	    &changed);
	if (error == 0 && changed != 0) {
		if (new_value <= 0 ||
		    new_value > FETH_LINK_LAYER_AGGRETATION_FACTOR_MAX) {
			return EINVAL;
		}
		if_fake_link_layer_aggregation_factor = new_value;
	}
	return error;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, link_layer_aggregation_factor,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_link_layer_aggregation_factor_sysctl, "IU",
    "Fake interface link layer aggregation factor");

#define FETH_TX_HEADROOM_MAX      32
static unsigned int if_fake_tx_headroom = FETH_TX_HEADROOM_MAX;
static int
feth_tx_headroom_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_tx_headroom,
	    sizeof(if_fake_tx_headroom), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > FETH_TX_HEADROOM_MAX ||
		    (new_value % 8) != 0) {
			return EINVAL;
		}
		if_fake_tx_headroom = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_headroom,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_tx_headroom_sysctl, "IU", "Fake ethernet Tx headroom");

static int if_fake_fcs = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, fcs, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_fcs, 0, "Fake interface using frame check sequence");

#define FETH_TRAILER_LENGTH_MAX 28
char feth_trailer[FETH_TRAILER_LENGTH_MAX + 1] = "trailertrailertrailertrailer";
static unsigned int if_fake_trailer_length = 0;
static int
feth_trailer_length_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_trailer_length,
	    sizeof(if_fake_trailer_length), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > FETH_TRAILER_LENGTH_MAX) {
			return EINVAL;
		}
		if_fake_trailer_length = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, trailer_length,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_trailer_length_sysctl, "IU", "Fake interface frame trailer length");

/* sysctl net.link.fake.max_mtu */
#define FETH_MAX_MTU_DEFAULT    2048
#define FETH_MAX_MTU_MAX        ((16 * 1024) - ETHER_HDR_LEN)

static unsigned int if_fake_max_mtu = FETH_MAX_MTU_DEFAULT;

/* sysctl net.link.fake.buflet_size */
#define FETH_BUFLET_SIZE_MIN            512
#define FETH_BUFLET_SIZE_MAX            (32 * 1024)
#define FETH_TSO_BUFLET_SIZE            (16 * 1024)

static unsigned int if_fake_buflet_size = FETH_BUFLET_SIZE_MIN;
static unsigned int if_fake_tso_buffer_size = FETH_TSO_BUFLET_SIZE;

static int
feth_tso_buffer_size_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_tso_buffer_size,
	    sizeof(if_fake_tso_buffer_size), &new_value, &changed);
	if (error == 0 && changed != 0) {
		/* must be a power of 2 between min and max */
		if (new_value > FETH_BUFLET_SIZE_MAX ||
		    new_value < FETH_BUFLET_SIZE_MIN ||
		    !is_power_of_two(new_value)) {
			return EINVAL;
		}
		if_fake_tso_buffer_size = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, tso_buf_size,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_tso_buffer_size_sysctl, "IU", "Fake interface TSO buffer size");

static int
feth_max_mtu_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_max_mtu,
	    sizeof(if_fake_max_mtu), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > FETH_MAX_MTU_MAX ||
		    new_value < ETHERMTU ||
		    new_value <= if_fake_buflet_size) {
			return EINVAL;
		}
		if_fake_max_mtu = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, max_mtu,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_max_mtu_sysctl, "IU", "Fake interface maximum MTU");

static int
feth_buflet_size_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_buflet_size,
	    sizeof(if_fake_buflet_size), &new_value, &changed);
	if (error == 0 && changed != 0) {
		/* must be a power of 2 between min and max */
		if (new_value > FETH_BUFLET_SIZE_MAX ||
		    new_value < FETH_BUFLET_SIZE_MIN ||
		    !is_power_of_two(new_value) ||
		    new_value >= if_fake_max_mtu) {
			return EINVAL;
		}
		if_fake_buflet_size = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, buflet_size,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_buflet_size_sysctl, "IU", "Fake interface buflet size");

static unsigned int if_fake_user_access = 0;

static int
feth_user_access_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_user_access,
	    sizeof(if_fake_user_access), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value != 0) {
			if (new_value != 1) {
				return EINVAL;
			}
		}
		if_fake_user_access = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, user_access,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED,
    0, 0, feth_user_access_sysctl, "IU", "Fake interface user access");

/* sysctl net.link.fake.if_adv_intvl (unit: millisecond) */
#define FETH_IF_ADV_INTVL_MIN            10
#define FETH_IF_ADV_INTVL_MAX            INT_MAX

static int if_fake_if_adv_interval = 0; /* no interface advisory */
static int
feth_if_adv_interval_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_if_adv_interval,
	    sizeof(if_fake_if_adv_interval), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if ((new_value != 0) && (new_value > FETH_IF_ADV_INTVL_MAX ||
		    new_value < FETH_IF_ADV_INTVL_MIN)) {
			return EINVAL;
		}
		if_fake_if_adv_interval = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, if_adv_intvl,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_if_adv_interval_sysctl, "IU",
    "Fake interface will generate interface advisories reports at the specified interval in ms");

/* sysctl net.link.fake.tx_drops */
/*
 * Fake ethernet will drop packet on the transmit path at the specified
 * rate, i.e drop one in every if_fake_tx_drops number of packets.
 */
#define FETH_TX_DROPS_MIN            0
#define FETH_TX_DROPS_MAX            INT_MAX
static int if_fake_tx_drops = 0; /* no packets are dropped */
static int
feth_fake_tx_drops_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_tx_drops,
	    sizeof(if_fake_tx_drops), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > FETH_TX_DROPS_MAX ||
		    new_value < FETH_TX_DROPS_MIN) {
			return EINVAL;
		}
		if_fake_tx_drops = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_drops,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_fake_tx_drops_sysctl, "IU",
    "Fake interface will intermittently drop packets on Tx path");

/* sysctl.net.link.fake.tx_exp_policy */

typedef enum {
	IFF_TX_EXP_POLICY_DISABLED = 0,          /* Expiry notification disabled */
	IFF_TX_EXP_POLICY_DROP_AND_NOTIFY = 1,   /* Expiry notification enabled; drop + notify mode */
	IFF_TX_EXP_POLICY_NOTIFY_ONLY = 2,       /* Expiry notification enabled; notify only mode */
	IFF_TX_EXP_POLICY_METADATA = 3,          /* Expiry notification enabled; use packet metadata */
} iff_tx_exp_policy_t;
static iff_tx_exp_policy_t if_fake_tx_exp_policy = IFF_TX_EXP_POLICY_DISABLED;

static int
feth_fake_tx_exp_policy_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_tx_exp_policy,
	    sizeof(if_fake_tx_exp_policy), &new_value, &changed);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
	    "if_fake_tx_exp_policy: %u -> %u (%d)",
	    if_fake_tx_exp_policy, new_value, changed);
	if (error == 0 && changed != 0) {
		if (new_value > IFF_TX_EXP_POLICY_METADATA ||
		    new_value < IFF_TX_EXP_POLICY_DISABLED) {
			return EINVAL;
		}
		if_fake_tx_exp_policy = new_value;
	}
	return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_exp_policy,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_fake_tx_exp_policy_sysctl, "IU",
    "Fake interface handling policy for expired TX attempts "
    "(0 disabled, 1 drop and notify, 2 notify only, 3 packet metadata)");

/* sysctl net.link.fake.tx_completion_mode */
typedef enum {
	IFF_TX_COMPL_MODE_SYNC = 0,
	IFF_TX_COMPL_MODE_ASYNC = 1,
} iff_tx_completion_mode_t;
static iff_tx_completion_mode_t if_tx_completion_mode = IFF_TX_COMPL_MODE_SYNC;
static int
feth_fake_tx_completion_mode_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_tx_completion_mode,
	    sizeof(if_tx_completion_mode), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > IFF_TX_COMPL_MODE_ASYNC ||
		    new_value < IFF_TX_COMPL_MODE_SYNC) {
			return EINVAL;
		}
		if_tx_completion_mode = new_value;
	}
	return 0;
}
SYSCTL_PROC(_net_link_fake, OID_AUTO, tx_completion_mode,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_fake_tx_completion_mode_sysctl, "IU",
    "Fake interface tx completion mode (0 synchronous, 1 asynchronous)");

/* sysctl net.link.fake.llink_cnt */

/* The maximum number of logical links (including default link) */
#define FETH_MAX_LLINKS 16
/*
 * The default number of logical links (including default link).
 * Zero means logical link mode is disabled.
 */
#define FETH_DEF_LLINKS 0

static uint32_t if_fake_llink_cnt = FETH_DEF_LLINKS;
static int
feth_fake_llink_cnt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_llink_cnt,
	    sizeof(if_fake_llink_cnt), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value > FETH_MAX_LLINKS) {
			return EINVAL;
		}
		if_fake_llink_cnt = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, llink_cnt,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_fake_llink_cnt_sysctl, "IU",
    "Fake interface logical link count");

/* sysctl net.link.fake.qset_cnt */

/* The maximum number of qsets for each logical link */
#define FETH_MAX_QSETS  16
/* The default number of qsets for each logical link */
#define FETH_DEF_QSETS  4

static uint32_t if_fake_qset_cnt = FETH_DEF_QSETS;
static int
feth_fake_qset_cnt_sysctl SYSCTL_HANDLER_ARGS
{
#pragma unused(oidp, arg1, arg2)
	unsigned int new_value;
	int changed;
	int error;

	error = sysctl_io_number(req, if_fake_qset_cnt,
	    sizeof(if_fake_qset_cnt), &new_value, &changed);
	if (error == 0 && changed != 0) {
		if (new_value == 0 ||
		    new_value > FETH_MAX_QSETS) {
			return EINVAL;
		}
		if_fake_qset_cnt = new_value;
	}
	return 0;
}

SYSCTL_PROC(_net_link_fake, OID_AUTO, qset_cnt,
    CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_LOCKED, 0, 0,
    feth_fake_qset_cnt_sysctl, "IU",
    "Fake interface queue set count");


static void
_mbuf_adjust_pkthdr_and_data(mbuf_t m, int len)
{
	mbuf_setdata(m, mtod(m, char *) + len, mbuf_len(m) - len);
	mbuf_pkthdr_adjustlen(m, -len);
}

static inline void *__indexable
get_bpf_header(mbuf_t m, struct ether_header * eh_p,
    struct ether_vlan_header * evl_p, size_t * header_len)
{
	void * header;

	/* no VLAN tag, just use the ethernet header */
	if ((m->m_pkthdr.csum_flags & CSUM_VLAN_TAG_VALID) == 0) {
		header = (struct ether_header *__bidi_indexable)eh_p;
		*header_len = sizeof(*eh_p);
		goto done;
	}

	/* has VLAN tag, populate the ether VLAN header */
	bcopy(eh_p, evl_p,
	    offsetof(struct ether_header, ether_type));   /* dst+src ether */
	evl_p->evl_encap_proto = htons(ETHERTYPE_VLAN);   /* VLAN encap */
	evl_p->evl_tag = htons(m->m_pkthdr.vlan_tag);     /* tag */
	evl_p->evl_proto = eh_p->ether_type;              /* proto */
	*header_len = sizeof(*evl_p);
	header = (struct ether_vlan_header *__bidi_indexable)evl_p;

done:
	return header;
}

typedef void (*_tap_func)(ifnet_t interface, u_int32_t dlt, mbuf_t packet,
    void *__sized_by(header_len) header, size_t header_len);

static void
fake_bpf_tap_common(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p,
    _tap_func func)
{
	struct ether_vlan_header        evl;
	void *                          header;
	size_t                          header_len;

	header = get_bpf_header(m, eh_p, &evl, &header_len);
	(*func)(ifp, DLT_EN10MB, m, header, header_len);
}

static inline void
fake_bpf_tap_in(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p)
{
	fake_bpf_tap_common(ifp, m, eh_p, bpf_tap_in);
}


static inline void
fake_bpf_tap_out(ifnet_t ifp, mbuf_t m, struct ether_header * eh_p)
{
	fake_bpf_tap_common(ifp, m, eh_p, bpf_tap_out);
}

/**
** virtual ethernet structures, types
**/

#define IFF_NUM_TX_RINGS_WMM_MODE       4
#define IFF_NUM_RX_RINGS_WMM_MODE       1
#define IFF_MAX_TX_RINGS        IFF_NUM_TX_RINGS_WMM_MODE
#define IFF_MAX_RX_RINGS        IFF_NUM_RX_RINGS_WMM_MODE
#define IFF_NUM_TX_QUEUES_WMM_MODE      4
#define IFF_NUM_RX_QUEUES_WMM_MODE      1
#define IFF_MAX_TX_QUEUES       IFF_NUM_TX_QUEUES_WMM_MODE
#define IFF_MAX_RX_QUEUES       IFF_NUM_RX_QUEUES_WMM_MODE

#define IFF_MAX_BATCH_SIZE 32

typedef uint16_t        iff_flags_t;
#define IFF_FLAGS_HWCSUM                0x0001
#define IFF_FLAGS_BSD_MODE              0x0002
#define IFF_FLAGS_DETACHING             0x0004
#define IFF_FLAGS_WMM_MODE              0x0008
#define IFF_FLAGS_MULTIBUFLETS          0x0010
#define IFF_FLAGS_TSO_SUPPORT           0x0020
#define IFF_FLAGS_LRO                   0x0040
#define IFF_FLAGS_VLAN_MTU              0x0080
#define IFF_FLAGS_VLAN_TAGGING          0x0100
#define IFF_FLAGS_SEPARATE_FRAME_HEADER 0x0200
#define IFF_FLAGS_NX_ATTACHED           0x0400

#if SKYWALK

typedef struct {
	uuid_t                  fnx_provider;
	uuid_t                  fnx_instance;
} fake_nx, *fake_nx_t;

typedef struct {
	kern_netif_queue_t      fq_queue;
} fake_queue;

typedef struct {
	kern_netif_qset_t       fqs_qset; /* provided by xnu */
	fake_queue              fqs_rx_queue[IFF_MAX_RX_QUEUES];
	fake_queue              fqs_tx_queue[IFF_MAX_TX_QUEUES];
	uint32_t                fqs_rx_queue_cnt;
	uint32_t                fqs_tx_queue_cnt;
	uint32_t                fqs_llink_idx;
	uint32_t                fqs_idx;
	uint32_t                fqs_dequeue_cnt;
	uint64_t                fqs_id;
	boolean_t               fqs_combined_mode;
} fake_qset;

typedef struct {
	uint64_t                fl_id;
	uint32_t                fl_idx;
	uint32_t                fl_qset_cnt;
	fake_qset               fl_qset[FETH_MAX_QSETS];
} fake_llink, * fake_llink_t;

static kern_pbufpool_t         S_pp;

#define IFF_TT_OUTPUT   0x01 /* generate trace_tag on output */
#define IFF_TT_INPUT    0x02 /* generate trace_tag on input */
static int if_fake_trace_tag_flags = 0;
SYSCTL_INT(_net_link_fake, OID_AUTO, trace_tag, CTLFLAG_RW | CTLFLAG_LOCKED,
    &if_fake_trace_tag_flags, 0, "Fake interface generate trace_tag");
static packet_trace_tag_t if_fake_trace_tag_current = 1;

#endif /* SKYWALK */

struct if_fake {
	char                    iff_name[IFNAMSIZ]; /* our unique id */
	ifnet_t                 iff_ifp;
	iff_flags_t             iff_flags;
	uint32_t                iff_retain_count;
	ifnet_t                 iff_peer;       /* the other end */
	int                     iff_media_current;
	int                     iff_media_active;
	uint32_t                iff_media_count;
	int                     iff_media_list[IF_FAKE_MEDIA_LIST_MAX];
	boolean_t               iff_start_busy;
	unsigned int            iff_max_mtu;
	uint32_t                iff_fcs;
	uint32_t                iff_trailer_length;
#if SKYWALK
	fake_nx                 iff_nx;
	struct netif_stats      *iff_nifs;
	uint32_t                iff_nifs_ref;
	uint32_t                iff_llink_cnt;
	kern_channel_ring_t     iff_rx_ring[IFF_MAX_RX_RINGS];
	kern_channel_ring_t     iff_tx_ring[IFF_MAX_TX_RINGS];
	fake_llink_t            iff_llink __counted_by_or_null(FETH_MAX_LLINKS);
	thread_call_t           iff_doorbell_tcall;
	thread_call_t           iff_if_adv_tcall;
	boolean_t               iff_doorbell_tcall_active;
	boolean_t               iff_waiting_for_tcall;
	boolean_t               iff_channel_connected;
	iff_pktpool_mode_t      iff_pp_mode;
	kern_pbufpool_t         iff_rx_pp;
	kern_pbufpool_t         iff_tx_pp;
	uint32_t                iff_tx_headroom;
	unsigned int            iff_adv_interval;
	uint32_t                iff_tx_drop_rate;
	uint32_t                iff_tx_pkts_count;
	iff_tx_completion_mode_t iff_tx_completion_mode;
	bool                    iff_intf_adv_enabled;
	void                    *iff_intf_adv_kern_ctx;
	kern_nexus_capab_interface_advisory_notify_fn_t iff_intf_adv_notify;
	iff_tx_exp_policy_t     iff_tx_exp_policy;
#endif /* SKYWALK */
};

typedef struct if_fake * __single if_fake_ref;

static if_fake_ref
ifnet_get_if_fake(ifnet_t ifp);

static inline boolean_t
feth_in_bsd_mode(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_BSD_MODE) != 0;
}

static inline void
feth_set_detaching(if_fake_ref fakeif)
{
	fakeif->iff_flags |= IFF_FLAGS_DETACHING;
}

static inline boolean_t
feth_is_detaching(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_DETACHING) != 0;
}

static int
feth_enable_dequeue_stall(ifnet_t ifp, uint32_t enable)
{
	int error;

	if (enable != 0) {
		error = ifnet_disable_output(ifp);
	} else {
		error = ifnet_enable_output(ifp);
	}

	return error;
}

#if SKYWALK
static inline boolean_t
feth_in_wmm_mode(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_WMM_MODE) != 0;
}

static inline boolean_t
feth_using_multibuflets(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_MULTIBUFLETS) != 0;
}
static void feth_detach_netif_nexus(if_fake_ref fakeif);

static inline boolean_t
feth_has_intf_advisory_configured(if_fake_ref fakeif)
{
	return fakeif->iff_adv_interval > 0;
}
#endif /* SKYWALK */

static inline bool
feth_supports_tso(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_TSO_SUPPORT) != 0;
}

static inline void
feth_set_supports_tso(if_fake_ref fakeif)
{
	fakeif->iff_flags |= IFF_FLAGS_TSO_SUPPORT;
}

static inline bool
feth_supports_vlan_mtu(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_VLAN_MTU) != 0;
}

static inline void
feth_set_supports_vlan_mtu(if_fake_ref fakeif)
{
	fakeif->iff_flags |= IFF_FLAGS_VLAN_MTU;
}

static inline bool
feth_supports_vlan_tagging(if_fake_ref fakeif)
{
	return (fakeif->iff_flags & IFF_FLAGS_VLAN_TAGGING) != 0;
}

static inline void
feth_set_supports_vlan_tagging(if_fake_ref fakeif)
{
	fakeif->iff_flags |= IFF_FLAGS_VLAN_TAGGING;
}


#define FETH_MAXUNIT    IF_MAXUNIT
#define FETH_ZONE_MAX_ELEM      MIN(IFNETS_MAX, FETH_MAXUNIT)

static  int feth_clone_create(struct if_clone *, u_int32_t, void *);
static  int feth_clone_destroy(ifnet_t);
static  int feth_output(ifnet_t ifp, struct mbuf *m);
static  void feth_start(ifnet_t ifp);
static  int feth_ioctl(ifnet_t ifp, u_long cmd, void * addr);
static  int feth_config(ifnet_t ifp, ifnet_t peer);
static  void feth_if_free(ifnet_t ifp);
static  void feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp);
static  void feth_free(if_fake_ref fakeif);

static struct if_clone
    feth_cloner = IF_CLONE_INITIALIZER(FAKE_ETHER_NAME,
    feth_clone_create,
    feth_clone_destroy,
    0,
    FETH_MAXUNIT);
static  void interface_link_event(ifnet_t ifp, u_int32_t event_code);

/* some media words to pretend to be ethernet */
#define FAKE_DEFAULT_MEDIA      IFM_MAKEWORD(IFM_ETHER, IFM_10G_T, IFM_FDX, 0)
static int default_media_words[] = {
	IFM_MAKEWORD(IFM_ETHER, 0, 0, 0),
	FAKE_DEFAULT_MEDIA,
	IFM_MAKEWORD(IFM_ETHER, IFM_2500_T, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_5000_T, IFM_FDX, 0),

	IFM_MAKEWORD(IFM_ETHER, IFM_10G_KX4, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_20G_KR2, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_2500_SX, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_25G_KR, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_40G_SR4, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_50G_CR2, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_56G_R4, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_100G_CR4, IFM_FDX, 0),
	IFM_MAKEWORD(IFM_ETHER, IFM_400G_AUI8, IFM_FDX, 0),
};
#define default_media_words_count (sizeof(default_media_words)          \
	                           / sizeof (default_media_words[0]))

/**
** veth locks
**/

static LCK_GRP_DECLARE(feth_lck_grp, "fake");
static LCK_MTX_DECLARE(feth_lck_mtx, &feth_lck_grp);

static inline void
feth_lock(void)
{
	lck_mtx_lock(&feth_lck_mtx);
}

static inline void
feth_unlock(void)
{
	lck_mtx_unlock(&feth_lck_mtx);
}

static inline int
get_max_mtu(int bsd_mode, unsigned int max_mtu)
{
	unsigned int    mtu;

	if (bsd_mode != 0) {
		mtu = (njcl > 0) ? (M16KCLBYTES - ETHER_HDR_LEN)
		    : MBIGCLBYTES - ETHER_HDR_LEN;
		if (mtu > max_mtu) {
			mtu = max_mtu;
		}
	} else {
		mtu = max_mtu;
	}
	return mtu;
}

static inline unsigned int
feth_max_mtu(ifnet_t ifp)
{
	if_fake_ref     fakeif;
	unsigned int    max_mtu = ETHERMTU;

	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif != NULL) {
		max_mtu = fakeif->iff_max_mtu;
	}
	feth_unlock();
	return max_mtu;
}

static void
feth_free(if_fake_ref fakeif)
{
	VERIFY(fakeif->iff_retain_count == 0);
#if SKYWALK
	if (!feth_in_bsd_mode(fakeif)) {
		if (fakeif->iff_pp_mode == IFF_PP_MODE_GLOBAL) {
			VERIFY(fakeif->iff_rx_pp == S_pp);
			VERIFY(fakeif->iff_tx_pp == S_pp);
			pp_release(fakeif->iff_rx_pp);
			fakeif->iff_rx_pp = NULL;
			pp_release(fakeif->iff_tx_pp);
			fakeif->iff_tx_pp = NULL;
			feth_lock();
			if (S_pp != NULL && S_pp->pp_refcnt == 1) {
				pp_release(S_pp);
				S_pp = NULL;
			}
			feth_unlock();
		} else {
			if (fakeif->iff_rx_pp != NULL) {
				pp_release(fakeif->iff_rx_pp);
				fakeif->iff_rx_pp = NULL;
			}
			if (fakeif->iff_tx_pp != NULL) {
				pp_release(fakeif->iff_tx_pp);
				fakeif->iff_tx_pp = NULL;
			}
		}
	}
#endif /* SKYWALK */

	FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s", fakeif->iff_name);
	if (fakeif->iff_llink != NULL) {
		fake_llink_t    llink;
		llink = fakeif->iff_llink;
		fakeif->iff_llink = NULL;
		kfree_type(fake_llink, FETH_MAX_LLINKS, llink);
	}
	kfree_type(struct if_fake, fakeif);
}

static void
feth_release(if_fake_ref fakeif)
{
	u_int32_t               old_retain_count;

	old_retain_count = OSDecrementAtomic(&fakeif->iff_retain_count);
	switch (old_retain_count) {
	case 0:
		VERIFY(old_retain_count != 0);
		break;
	case 1:
		feth_free(fakeif);
		break;
	default:
		break;
	}
	return;
}

#if SKYWALK

static void
feth_retain(if_fake_ref fakeif)
{
	OSIncrementAtomic(&fakeif->iff_retain_count);
}

static void
feth_packet_pool_init_prepare(if_fake_ref fakeif,
    struct kern_pbufpool_init *pp_init)
{
	uint32_t max_mtu = fakeif->iff_max_mtu;
	uint32_t buflet_size = if_fake_buflet_size;

	bzero(pp_init, sizeof(*pp_init));
	pp_init->kbi_version = KERN_PBUFPOOL_CURRENT_VERSION;
	pp_init->kbi_flags |= KBIF_VIRTUAL_DEVICE;
	pp_init->kbi_packets = 1024; /* TBD configurable */
	if (feth_supports_tso(fakeif)) {
		buflet_size = if_fake_tso_buffer_size;
	}
	if (feth_using_multibuflets(fakeif)) {
		pp_init->kbi_bufsize = buflet_size;
		pp_init->kbi_max_frags = howmany(max_mtu, buflet_size);
		pp_init->kbi_buflets = pp_init->kbi_packets *
		    pp_init->kbi_max_frags;
		pp_init->kbi_flags |= KBIF_BUFFER_ON_DEMAND;
	} else {
		pp_init->kbi_bufsize = max(max_mtu, buflet_size);
		pp_init->kbi_max_frags = 1;
		pp_init->kbi_buflets = pp_init->kbi_packets;
	}
	pp_init->kbi_buf_seg_size = skmem_usr_buf_seg_size;
	if (if_fake_user_access != 0) {
		pp_init->kbi_flags |= KBIF_USER_ACCESS;
	}
	pp_init->kbi_ctx = NULL;
	pp_init->kbi_ctx_retain = NULL;
	pp_init->kbi_ctx_release = NULL;
}

static errno_t
feth_packet_pool_make(if_fake_ref fakeif)
{
	struct kern_pbufpool_init pp_init;
	errno_t err;

	feth_packet_pool_init_prepare(fakeif, &pp_init);

	switch (fakeif->iff_pp_mode) {
	case IFF_PP_MODE_GLOBAL:
		feth_lock();
		if (S_pp == NULL) {
			(void)snprintf((char *)pp_init.kbi_name,
			    sizeof(pp_init.kbi_name), "%s", "feth shared pp");
			err = kern_pbufpool_create(&pp_init, &S_pp, NULL);
		}
		pp_retain(S_pp);
		feth_unlock();
		fakeif->iff_rx_pp = S_pp;
		pp_retain(S_pp);
		fakeif->iff_tx_pp = S_pp;
		break;
	case IFF_PP_MODE_PRIVATE:
		(void)snprintf((char *)pp_init.kbi_name,
		    sizeof(pp_init.kbi_name), "%s pp", fakeif->iff_name);
		err = kern_pbufpool_create(&pp_init, &fakeif->iff_rx_pp, NULL);
		pp_retain(fakeif->iff_rx_pp);
		fakeif->iff_tx_pp = fakeif->iff_rx_pp;
		break;
	case IFF_PP_MODE_PRIVATE_SPLIT:
		(void)snprintf((char *)pp_init.kbi_name,
		    sizeof(pp_init.kbi_name), "%s rx pp", fakeif->iff_name);
		pp_init.kbi_flags &= ~(KBIF_IODIR_IN | KBIF_IODIR_OUT |
		    KBIF_BUFFER_ON_DEMAND | KBIF_KERNEL_READONLY);
		pp_init.kbi_flags |= (KBIF_IODIR_IN | KBIF_BUFFER_ON_DEMAND);
		pp_init.kbi_packets = 1024;
		pp_init.kbi_bufsize = if_fake_link_layer_aggregation_factor * 1024;
		err = kern_pbufpool_create(&pp_init, &fakeif->iff_rx_pp, NULL);
		if (err != 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "rx pp create failed %d", err);
			return err;
		}
		pp_init.kbi_flags &= ~(KBIF_IODIR_IN | KBIF_IODIR_OUT |
		    KBIF_BUFFER_ON_DEMAND | KBIF_KERNEL_READONLY);
		pp_init.kbi_flags |= KBIF_IODIR_OUT;
		pp_init.kbi_packets = 1024;            /* TBD configurable */
		pp_init.kbi_bufsize = fakeif->iff_max_mtu;
		(void)snprintf((char *)pp_init.kbi_name,
		    sizeof(pp_init.kbi_name), "%s tx pp", fakeif->iff_name);
		err = kern_pbufpool_create(&pp_init, &fakeif->iff_tx_pp, NULL);
		if (err != 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "tx pp create failed %d", err);
			pp_release(fakeif->iff_rx_pp);
			return err;
		}
		break;
	default:
		VERIFY(0);
		__builtin_unreachable();
	}

	return 0;
}

static void
feth_packet_set_trace_tag(kern_packet_t ph, int flag)
{
	if (if_fake_trace_tag_flags & flag) {
		if (++if_fake_trace_tag_current == 0) {
			if_fake_trace_tag_current = 1;
		}
		kern_packet_set_trace_tag(ph, if_fake_trace_tag_current);
	}
}

static errno_t
feth_clone_packet(if_fake_ref dif, kern_packet_t sph, kern_packet_t *pdph)
{
	errno_t err = 0;
	kern_pbufpool_t pp = dif->iff_rx_pp;
	kern_packet_t dph = 0, dph0 = 0;
	kern_buflet_t sbuf, dbuf0 = NULL, dbuf;
	caddr_t saddr, daddr;
	uint32_t soff, doff;
	uint32_t slen, dlen;
	uint32_t dlim0, dlim;

	sbuf = kern_packet_get_next_buflet(sph, NULL);
	saddr = __unsafe_forge_bidi_indexable(caddr_t,
	    kern_buflet_get_data_address(sbuf),
	    kern_buflet_get_data_limit(sbuf));
	doff = soff = kern_buflet_get_data_offset(sbuf);
	dlen = slen = kern_buflet_get_data_length(sbuf);

	/* packet clone is only supported for single-buflet */
	ASSERT(kern_packet_get_buflet_count(sph) == 1);
	ASSERT(soff == kern_packet_get_headroom(sph));
	ASSERT(slen == kern_packet_get_data_length(sph));

	dph0 = *pdph;
	if (dph0 == 0) {
		dlim0 = 0;
	} else {
		dbuf0 = kern_packet_get_next_buflet(dph0, NULL);
		ASSERT(kern_buflet_get_object_limit(dbuf0) ==
		    PP_BUF_OBJ_SIZE_DEF(pp));
		ASSERT(kern_buflet_get_data_limit(dbuf0) % 16 == 0);
		dlim0 = ((size_t)kern_buflet_get_object_address(dbuf0) +
		    kern_buflet_get_object_limit(dbuf0)) -
		    ((size_t)kern_buflet_get_data_address(dbuf0) +
		    kern_buflet_get_data_limit(dbuf0));
	}

	if (doff + dlen > dlim0) {
		err = kern_pbufpool_alloc_nosleep(pp, 1, &dph);
		if (err != 0) {
			STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
			STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_PKT);
			return err;
		}
		dbuf = kern_packet_get_next_buflet(dph, NULL);
		ASSERT(kern_buflet_get_data_address(dbuf) ==
		    kern_buflet_get_object_address(dbuf));
		daddr = __unsafe_forge_bidi_indexable(caddr_t,
		    kern_buflet_get_data_address(dbuf),
		    kern_buflet_get_data_limit(dbuf));
		dlim = kern_buflet_get_object_limit(dbuf);
		ASSERT(dlim == PP_BUF_OBJ_SIZE_DEF(pp));
	} else {
		err = kern_packet_clone_nosleep(dph0, &dph, KPKT_COPY_LIGHT);
		if (err != 0) {
			FAKE_LOG(LOG_INFO, FE_DBGF_OUTPUT,
			    "packet clone err %d", err);
			return err;
		}
		dbuf = kern_packet_get_next_buflet(dph, NULL);
		ASSERT(kern_buflet_get_object_address(dbuf) ==
		    kern_buflet_get_object_address(dbuf0));
		daddr = __unsafe_forge_bidi_indexable(caddr_t,
		    kern_buflet_get_data_address(dbuf0),
		    kern_buflet_get_object_limit(dbuf0)) + kern_buflet_get_data_limit(dbuf0);
		dlim = dlim0;
	}

	ASSERT(doff + dlen <= dlim);

	ASSERT((uintptr_t)daddr % 16 == 0);
	bcopy(saddr + soff, daddr + doff, slen);

	dlim = MIN(dlim, P2ROUNDUP(doff + dlen, 16));
	err = kern_buflet_set_data_address(dbuf, daddr);
	VERIFY(err == 0);
	err = kern_buflet_set_data_limit(dbuf, dlim);
	VERIFY(err == 0);
	err = kern_buflet_set_data_length(dbuf, dlen);
	VERIFY(err == 0);
	err = kern_buflet_set_data_offset(dbuf, doff);
	VERIFY(err == 0);
	err = kern_packet_set_headroom(dph, doff);
	VERIFY(err == 0);
	err = kern_packet_set_link_header_length(dph,
	    kern_packet_get_link_header_length(sph));
	VERIFY(err == 0);
	err = kern_packet_set_service_class(dph,
	    kern_packet_get_service_class(sph));
	VERIFY(err == 0);
	err = kern_packet_finalize(dph);
	VERIFY(err == 0);
	*pdph = dph;

	return err;
}

static inline void
feth_copy_buflet(kern_buflet_t sbuf, kern_buflet_t dbuf)
{
	errno_t err;
	uint32_t off, len;
	caddr_t saddr, daddr;

	saddr = __unsafe_forge_bidi_indexable(caddr_t,
	    kern_buflet_get_data_address(sbuf),
	    kern_buflet_get_data_limit(sbuf));
	off = kern_buflet_get_data_offset(sbuf);
	len = kern_buflet_get_data_length(sbuf);
	daddr = __unsafe_forge_bidi_indexable(caddr_t,
	    kern_buflet_get_data_address(dbuf),
	    kern_buflet_get_data_limit(dbuf));
	bcopy(saddr + off, daddr + off, len);
	err = kern_buflet_set_data_offset(dbuf, off);
	VERIFY(err == 0);
	err = kern_buflet_set_data_length(dbuf, len);
	VERIFY(err == 0);
}

static int
feth_add_packet_trailer(kern_packet_t ph, void * __sized_by(trailer_len) trailer, size_t trailer_len)
{
	errno_t err = 0;

	ASSERT(trailer_len <= FETH_TRAILER_LENGTH_MAX);

	kern_buflet_t buf = NULL, iter = NULL;
	while ((iter = kern_packet_get_next_buflet(ph, iter)) != NULL) {
		buf = iter;
	}
	ASSERT(buf != NULL);

	uint32_t dlim = kern_buflet_get_data_limit(buf);
	uint32_t doff = kern_buflet_get_data_offset(buf);
	uint32_t dlen = kern_buflet_get_data_length(buf);

	size_t trailer_room = dlim - doff - dlen;

	if (trailer_room < trailer_len) {
		FAKE_LOG(LOG_INFO, FE_DBGF_OUTPUT, "not enough room");
		return ERANGE;
	}

	void *data = __unsafe_forge_bidi_indexable(caddr_t,
	    kern_buflet_get_data_address(buf),
	    kern_buflet_get_data_limit(buf)) + doff + dlen;
	memcpy(data, trailer, trailer_len);

	err = kern_buflet_set_data_length(buf, dlen + trailer_len);
	VERIFY(err == 0);

	err = kern_packet_finalize(ph);
	VERIFY(err == 0);

	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%zuB trailer added", trailer_len);

	return 0;
}

static int
feth_add_packet_fcs(kern_packet_t ph)
{
	uint32_t crc = 0;
	int err;

	ASSERT(sizeof(crc) == ETHER_CRC_LEN);

	kern_buflet_t buf = NULL;
	while ((buf = kern_packet_get_next_buflet(ph, buf)) != NULL) {
		uint32_t doff = kern_buflet_get_data_offset(buf);
		uint32_t dlen = kern_buflet_get_data_length(buf);
		void *data = __unsafe_forge_bidi_indexable(caddr_t,
		    kern_buflet_get_data_address(buf),
		    kern_buflet_get_data_limit(buf)) + doff;
		crc = crc32(crc, data, dlen);
	}

	err = feth_add_packet_trailer(ph, &crc, ETHER_CRC_LEN);
	if (!err) {
		return err;
	}

	err = kern_packet_set_link_ethfcs(ph);
	VERIFY(err == 0);

	return 0;
}

static errno_t
feth_copy_packet(if_fake_ref dif, kern_packet_t sph, kern_packet_t *pdph)
{
	errno_t err = 0;
	uint16_t i, bufcnt;
	mach_vm_address_t baddr;
	kern_buflet_t sbuf = NULL, dbuf = NULL;
	kern_pbufpool_t pp = dif->iff_rx_pp;
	kern_packet_t dph;
	boolean_t multi_buflet = feth_using_multibuflets(dif);

	bufcnt = kern_packet_get_buflet_count(sph);
	ASSERT((bufcnt == 1) || multi_buflet);
	*pdph = 0;

	err = kern_pbufpool_alloc_nosleep(pp, 1, &dph);
	if (err != 0) {
		STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
		STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_PKT);
		return err;
	}

	/* pre-constructed single buflet packet copy */
	sbuf = kern_packet_get_next_buflet(sph, NULL);
	dbuf = kern_packet_get_next_buflet(dph, NULL);
	feth_copy_buflet(sbuf, dbuf);

	if (!multi_buflet) {
		goto done;
	}

	/* un-constructed multi-buflet packet copy */
	for (i = 1; i < bufcnt; i++) {
		kern_buflet_t __single dbuf_next = NULL;

		sbuf = kern_packet_get_next_buflet(sph, sbuf);
		VERIFY(sbuf != NULL);
		err = kern_pbufpool_alloc_buflet_nosleep(pp, &dbuf_next);
		if (err != 0) {
			STATS_INC(dif->iff_nifs, NETIF_STATS_DROP);
			STATS_INC(dif->iff_nifs, NETIF_STATS_DROP_NOMEM_BUF);
			break;
		}
		ASSERT(dbuf_next != NULL);
		feth_copy_buflet(sbuf, dbuf_next);
		err = kern_packet_add_buflet(dph, dbuf, dbuf_next);
		VERIFY(err == 0);
		dbuf = dbuf_next;
	}
	if (__improbable(err != 0)) {
		dbuf = NULL;
		while (i-- != 0) {
			dbuf = kern_packet_get_next_buflet(dph, dbuf);
			VERIFY(dbuf != NULL);
			baddr = (mach_vm_address_t)
			    kern_buflet_get_data_address(dbuf);
			VERIFY(baddr != 0);
		}
		kern_pbufpool_free(pp, dph);
		dph = 0;
	}

done:
	if (__probable(err == 0)) {
		err = kern_packet_set_headroom(dph,
		    kern_packet_get_headroom(sph));
		VERIFY(err == 0);
		err = kern_packet_set_link_header_length(dph,
		    kern_packet_get_link_header_length(sph));
		VERIFY(err == 0);
		err = kern_packet_set_service_class(dph,
		    kern_packet_get_service_class(sph));
		VERIFY(err == 0);
		err = kern_packet_finalize(dph);
		VERIFY(err == 0);
		VERIFY(bufcnt == kern_packet_get_buflet_count(dph));
		*pdph = dph;
	}
	return err;
}

static inline void
feth_update_pkt_tso_metadata_for_rx(kern_packet_t ph)
{
	/*
	 * Nothing to do if not a TSO offloaded packet.
	 */
	uint16_t seg_sz = 0;
	seg_sz = kern_packet_get_protocol_segment_size(ph);
	if (seg_sz == 0) {
		return;
	}
	/*
	 * For RX, make the packet appear as a fully validated LRO packet.
	 */
	packet_csum_flags_t csum_flags = PACKET_CSUM_IP_CHECKED |
	    PACKET_CSUM_IP_VALID | PACKET_CSUM_DATA_VALID |
	    PACKET_CSUM_PSEUDO_HDR;
	(void) kern_packet_set_inet_checksum(ph, csum_flags, 0, 0xFFFF, FALSE);
	return;
}

static void
feth_rx_submit(if_fake_ref sif, if_fake_ref dif, kern_packet_t * __counted_by(n_pkts) sphs,
    uint32_t n_pkts)
{
	errno_t err = 0;
	struct kern_channel_ring_stat_increment stats;
	kern_channel_ring_t rx_ring = NULL;
	kern_channel_slot_t rx_slot = NULL, last_rx_slot = NULL;
	kern_packet_t sph = 0, dph = 0;

	memset(&stats, 0, sizeof(stats));

	rx_ring = dif->iff_rx_ring[0];
	if (rx_ring == NULL) {
		return;
	}

	kr_enter(rx_ring, TRUE);
	kern_channel_reclaim(rx_ring);
	rx_slot = kern_channel_get_next_slot(rx_ring, NULL, NULL);

	for (uint32_t i = 0; i < n_pkts && rx_slot != NULL; i++) {
		sph = sphs[i];

		switch (dif->iff_pp_mode) {
		case IFF_PP_MODE_GLOBAL:
			sphs[i] = 0;
			dph = sph;
			feth_update_pkt_tso_metadata_for_rx(dph);
			err = kern_packet_finalize(dph);
			VERIFY(err == 0);
			break;
		case IFF_PP_MODE_PRIVATE:
			err = feth_copy_packet(dif, sph, &dph);
			break;
		case IFF_PP_MODE_PRIVATE_SPLIT:
			err = feth_clone_packet(dif, sph, &dph);
			break;
		default:
			VERIFY(0);
			__builtin_unreachable();
		}
		if (__improbable(err != 0)) {
			continue;
		}

		if (sif->iff_trailer_length != 0) {
			feth_add_packet_trailer(dph, feth_trailer,
			    sif->iff_trailer_length);
		}
		if (sif->iff_fcs != 0) {
			feth_add_packet_fcs(dph);
		}
		feth_packet_set_trace_tag(dph, IFF_TT_INPUT);
		bpf_tap_packet_in(dif->iff_ifp, DLT_EN10MB, dph, NULL, 0);
		stats.kcrsi_slots_transferred++;
		stats.kcrsi_bytes_transferred
		        += kern_packet_get_data_length(dph);

		/* attach the packet to the RX ring */
		err = kern_channel_slot_attach_packet(rx_ring, rx_slot, dph);
		VERIFY(err == 0);
		last_rx_slot = rx_slot;
		rx_slot = kern_channel_get_next_slot(rx_ring, rx_slot, NULL);
	}

	if (last_rx_slot != NULL) {
		kern_channel_advance_slot(rx_ring, last_rx_slot);
		kern_channel_increment_ring_net_stats(rx_ring, dif->iff_ifp,
		    &stats);
	}

	if (rx_ring != NULL) {
		kr_exit(rx_ring);
		kern_channel_notify(rx_ring, 0);
	}
}

static void
feth_rx_queue_submit(if_fake_ref sif, if_fake_ref dif, uint32_t llink_idx,
    uint32_t qset_idx, kern_packet_t * __counted_by(n_pkts) sphs, uint32_t n_pkts)
{
	errno_t err = 0;
	kern_netif_queue_t queue;
	kern_packet_t sph = 0, dph = 0;
	fake_llink *llink;
	fake_qset *qset;

	if (llink_idx >= dif->iff_llink_cnt) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
		    "invalid llink_idx idx %d (max %d) on peer %s",
		    llink_idx, dif->iff_llink_cnt, dif->iff_name);
		return;
	}
	llink = &dif->iff_llink[llink_idx];
	if (qset_idx >= llink->fl_qset_cnt) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
		    "invalid qset_idx %d (max %d) on peer %s",
		    qset_idx, llink->fl_qset_cnt, dif->iff_name);
		return;
	}
	qset = &dif->iff_llink[llink_idx].fl_qset[qset_idx];
	queue = qset->fqs_rx_queue[0].fq_queue;
	if (queue == NULL) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT,
		    "NULL default queue (llink_idx %d, qset_idx %d) on peer %s",
		    llink_idx, qset_idx, dif->iff_name);
		return;
	}
	for (uint32_t i = 0; i < n_pkts; i++) {
		uint32_t flags;

		sph = sphs[i];

		switch (dif->iff_pp_mode) {
		case IFF_PP_MODE_GLOBAL:
			sphs[i] = 0;
			dph = sph;
			feth_update_pkt_tso_metadata_for_rx(dph);
			break;
		case IFF_PP_MODE_PRIVATE:
			err = feth_copy_packet(dif, sph, &dph);
			break;
		case IFF_PP_MODE_PRIVATE_SPLIT:
			err = feth_clone_packet(dif, sph, &dph);
			break;
		default:
			VERIFY(0);
			__builtin_unreachable();
		}
		if (__improbable(err != 0)) {
			continue;
		}

		if (sif->iff_trailer_length != 0) {
			feth_add_packet_trailer(dph, feth_trailer,
			    sif->iff_trailer_length);
		}
		if (sif->iff_fcs != 0) {
			feth_add_packet_fcs(dph);
		}
		feth_packet_set_trace_tag(dph, IFF_TT_INPUT);
		bpf_tap_packet_in(dif->iff_ifp, DLT_EN10MB, dph, NULL, 0);

		flags = (i == n_pkts - 1) ?
		    KERN_NETIF_QUEUE_RX_ENQUEUE_FLAG_FLUSH : 0;
		kern_netif_queue_rx_enqueue(queue, dph, 1, flags);
	}
}

static void
feth_tx_complete(if_fake_ref fakeif, kern_packet_t * __counted_by(nphs) phs, uint32_t nphs)
{
	for (uint32_t i = 0; i < nphs; i++) {
		kern_packet_t ph = phs[i];
		if (ph == 0) {
			continue;
		}
		int err = kern_packet_set_tx_completion_status(ph, 0);
		VERIFY(err == 0);
		kern_packet_tx_completion(ph, fakeif->iff_ifp);
		kern_pbufpool_free(fakeif->iff_tx_pp, phs[i]);
		phs[i] = 0;
	}
}

#define NSEC_PER_USEC 1000ull
/*
 * Calculate the time delta that passed from `since' to `until'.
 * If `until' happens before `since', returns negative value.
 */
static bool
feth_packet_has_expired(if_fake_ref __unused fakeif, kern_packet_t ph,
    uint64_t *out_deadline)
{
	uint64_t now;
	uint64_t packet_expire_time_mach;
	int64_t time_until_expiration;
	errno_t err;
	bool expired = false;

	static mach_timebase_info_data_t clock_timebase = {0, 0};

	if (clock_timebase.denom == 0) {
		clock_timebase_info(&clock_timebase);
		VERIFY(clock_timebase.denom != 0);
	}

	err = kern_packet_get_expire_time(ph, &packet_expire_time_mach);
	if (err) {
		goto out;
	}

	now = mach_absolute_time();
	time_until_expiration = packet_expire_time_mach - now;
	if (time_until_expiration < 0) {
		/* The packet had expired */
		expired = true;
		goto out;
	}

	/* Convert the time_delta from mach ticks to nanoseconds */
	time_until_expiration *= clock_timebase.numer;
	time_until_expiration /= clock_timebase.denom;
	/* convert from nanoseconds to microseconds */
	time_until_expiration /= 1000ull;

	if (if_fake_expiration_threshold_us < time_until_expiration) {
		/* packet has some life ahead of it */
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "Packet has %llu usec until expiration",
		    time_until_expiration);
		goto out;
	}

out:
	if (expired && out_deadline) {
		*out_deadline = packet_expire_time_mach;
	}

	return expired;
}

static errno_t
feth_get_packet_notification_details(if_fake_ref fakeif, kern_packet_t ph,
    packet_id_t *pkt_id, uint32_t *nx_port_id)
{
	errno_t err = 0;

	err = kern_packet_get_packetid(ph, pkt_id);
	if (err != 0) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "%s err=%d getting packetid", fakeif->iff_name, err);
		return err;
	}

	err = kern_packet_get_tx_nexus_port_id(ph, nx_port_id);
	if (err != 0) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "%s err=%d getting nx_port_id", fakeif->iff_name, err);
		return err;
	}

	return 0;
}

static packet_expiry_action_t
feth_get_effective_expn_action(if_fake_ref fakeif, kern_packet_t ph)
{
	errno_t err;
	packet_expiry_action_t expiry_action;

	switch (fakeif->iff_tx_exp_policy) {
	case IFF_TX_EXP_POLICY_DISABLED:
		expiry_action = PACKET_EXPIRY_ACTION_NONE;
		break;
	case IFF_TX_EXP_POLICY_NOTIFY_ONLY:
		expiry_action = PACKET_EXPIRY_ACTION_NOTIFY;
		break;
	case IFF_TX_EXP_POLICY_DROP_AND_NOTIFY:
		expiry_action = PACKET_EXPIRY_ACTION_DROP;
		break;
	case IFF_TX_EXP_POLICY_METADATA:
		err = kern_packet_get_expiry_action(ph, &expiry_action);
		if (err != 0) {
			if (err != ENOENT) {
				FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
				    "Error %d when getting expiry action",
				    err);
			}
			expiry_action = PACKET_EXPIRY_ACTION_NONE;
		}
		break;
	default:
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "Unrecognized value %d for \"net.link.fake.tx_exp_policy\"",
		    fakeif->iff_tx_exp_policy);
		expiry_action = PACKET_EXPIRY_ACTION_NONE;
	}

	return expiry_action;
}

/* returns true if the packet is selected for epxiration and should be dropped */
static bool
feth_tx_expired_error(if_fake_ref fakeif, kern_packet_t ph)
{
	int err = 0;
	uint32_t nx_port_id = 0;
	os_channel_event_packet_transmit_expired_t expn = {0};
	packet_expiry_action_t expiry_action = PACKET_EXPIRY_ACTION_NONE;

	FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC, "%s", fakeif->iff_name);

	if (feth_packet_has_expired(fakeif, ph, &expn.packet_tx_expiration_deadline)) {
		expiry_action = feth_get_effective_expn_action(fakeif, ph);
	}

	bool drop_packet = (expiry_action == PACKET_EXPIRY_ACTION_DROP);
	if (expiry_action != PACKET_EXPIRY_ACTION_NONE) {
		/* set the expiration status code */
		expn.packet_tx_expiration_status = drop_packet ?
		    CHANNEL_EVENT_PKT_TRANSMIT_EXPIRED_ERR_EXPIRED_DROPPED :
		    CHANNEL_EVENT_PKT_TRANSMIT_EXPIRED_ERR_EXPIRED_NOT_DROPPED;

		/* Mark the expiration timestamp */
		expn.packet_tx_expiration_timestamp = mach_absolute_time();

		err = feth_get_packet_notification_details(fakeif, ph,
		    &expn.packet_id, &nx_port_id);

		if (err == 0) {
			err = kern_channel_event_transmit_expired(
				fakeif->iff_ifp, &expn, nx_port_id);
			FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
			    "%s sent expiry notification on nexus port "
			    "%u notif code %u",
			    fakeif->iff_name, nx_port_id,
			    expn.packet_tx_expiration_status);
		}
		if (err != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
			    "%s err=%d, nx_port_id: 0x%x",
			    fakeif->iff_name, err, nx_port_id);
		}
	}

	return drop_packet;
}

/* returns true if the packet is selected for TX error & dropped */
static bool
feth_tx_complete_error(if_fake_ref fakeif, kern_packet_t ph)
{
	int err;

	if (fakeif->iff_tx_drop_rate == 0 ||
	    fakeif->iff_tx_pkts_count != fakeif->iff_tx_drop_rate) {
		return false;
	}
	/* simulate TX completion error on the packet */
	if (fakeif->iff_tx_completion_mode == IFF_TX_COMPL_MODE_SYNC) {
		err = kern_packet_set_tx_completion_status(ph,
		    CHANNEL_EVENT_PKT_TRANSMIT_STATUS_ERR_RETRY_FAILED);
		VERIFY(err == 0);
		kern_packet_tx_completion(ph, fakeif->iff_ifp);
	} else {
		uint32_t nx_port_id = 0;
		os_channel_event_packet_transmit_status_t pkt_tx_status = {0};

		pkt_tx_status.packet_status =
		    CHANNEL_EVENT_PKT_TRANSMIT_STATUS_ERR_RETRY_FAILED;
		err = feth_get_packet_notification_details(fakeif, ph,
		    &pkt_tx_status.packet_id, &nx_port_id);
		if (err == 0) {
			err = kern_channel_event_transmit_status(
				fakeif->iff_ifp, &pkt_tx_status, nx_port_id);
		}
		if (err != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
			    "%s err=%d, nx_port_id: 0x%x",
			    fakeif->iff_name, err, nx_port_id);
		}
	}

	return true;
}

static void
feth_if_adv(thread_call_param_t arg0, thread_call_param_t arg1)
{
#pragma unused(arg1)
	errno_t                            error;
	if_fake_ref                        fakeif = (if_fake_ref)arg0;
	struct ifnet_interface_advisory    if_adv;
	struct ifnet_stats_param           if_stat;

	feth_lock();
	if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
		feth_unlock();
		return;
	}
	feth_unlock();

	if (!fakeif->iff_intf_adv_enabled) {
		goto done;
	}

	error = ifnet_stat(fakeif->iff_ifp, &if_stat);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, 0, "%s: ifnet_stat() failed %d",
		    fakeif->iff_name, error);
		goto done;
	}
	if_adv.header.version = IF_INTERFACE_ADVISORY_VERSION_CURRENT;
	if_adv.header.direction = IF_INTERFACE_ADVISORY_DIRECTION_TX;
	if_adv.header.interface_type =
	    IF_INTERFACE_ADVISORY_INTERFACE_TYPE_WIFI;
	if_adv.capacity.timestamp = mach_absolute_time();
	if_adv.capacity.rate_trend_suggestion =
	    IF_INTERFACE_ADVISORY_RATE_SUGGESTION_RAMP_NEUTRAL;
	if_adv.capacity.max_bandwidth = 1000 * 1000 * 1000; /* 1Gbps */
	if_adv.capacity.total_byte_count = if_stat.packets_out;
	if_adv.capacity.average_throughput = 1000 * 1000 * 1000; /* 1Gbps */
	if_adv.capacity.flushable_queue_size = UINT32_MAX;
	if_adv.capacity.non_flushable_queue_size = UINT32_MAX;
	if_adv.capacity.average_delay = 1; /* ms */

	error = fakeif->iff_intf_adv_notify(fakeif->iff_intf_adv_kern_ctx,
	    &if_adv);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, 0,
		    "%s: interface advisory report failed %d",
		    fakeif->iff_name, error);
	}

done:
	feth_lock();
	if (!feth_is_detaching(fakeif) && fakeif->iff_channel_connected) {
		uint64_t deadline;
		clock_interval_to_deadline(fakeif->iff_adv_interval,
		    NSEC_PER_MSEC, &deadline);
		thread_call_enter_delayed(fakeif->iff_if_adv_tcall, deadline);
	}
	feth_unlock();
}

static int
feth_if_adv_tcall_create(if_fake_ref fakeif)
{
	uint64_t deadline;

	feth_lock();
	ASSERT(fakeif->iff_if_adv_tcall == NULL);
	ASSERT(fakeif->iff_adv_interval > 0);
	ASSERT(fakeif->iff_channel_connected);
	fakeif->iff_if_adv_tcall =
	    thread_call_allocate_with_options(feth_if_adv,
	    (thread_call_param_t)fakeif, THREAD_CALL_PRIORITY_KERNEL,
	    THREAD_CALL_OPTIONS_ONCE);
	if (fakeif->iff_if_adv_tcall == NULL) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "%s if_adv tcall alloc failed",
		    fakeif->iff_name);
		feth_unlock();
		return ENXIO;
	}
	/* retain for the interface advisory thread call */
	feth_retain(fakeif);
	clock_interval_to_deadline(fakeif->iff_adv_interval,
	    NSEC_PER_MSEC, &deadline);
	thread_call_enter_delayed(fakeif->iff_if_adv_tcall, deadline);
	feth_unlock();
	return 0;
}

/**
** nexus netif domain provider
**/
static errno_t
feth_nxdp_init(kern_nexus_domain_provider_t domprov)
{
#pragma unused(domprov)
	return 0;
}

static void
feth_nxdp_fini(kern_nexus_domain_provider_t domprov)
{
#pragma unused(domprov)
}

static uuid_t                   feth_nx_dom_prov;

static errno_t
feth_register_nexus_domain_provider(void)
{
	const struct kern_nexus_domain_provider_init dp_init = {
		.nxdpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
		.nxdpi_flags = 0,
		.nxdpi_init = feth_nxdp_init,
		.nxdpi_fini = feth_nxdp_fini
	};
	errno_t                         err = 0;

	nexus_domain_provider_name_t feth_provider_name = "com.apple.feth";

	/* feth_nxdp_init() is called before this function returns */
	err = kern_nexus_register_domain_provider(NEXUS_TYPE_NET_IF,
	    feth_provider_name,
	    &dp_init, sizeof(dp_init),
	    &feth_nx_dom_prov);
	if (err != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "failed to register domain provider");
		return err;
	}
	return 0;
}

/**
** netif nexus routines
**/
static if_fake_ref
feth_nexus_context(kern_nexus_t nexus)
{
	if_fake_ref fakeif;

	fakeif = (if_fake_ref)kern_nexus_get_context(nexus);
	assert(fakeif != NULL);
	return fakeif;
}

static uint8_t
feth_find_tx_ring_by_svc(kern_packet_svc_class_t svc_class)
{
	switch (svc_class) {
	case KPKT_SC_VO:
		return 0;
	case KPKT_SC_VI:
		return 1;
	case KPKT_SC_BE:
		return 2;
	case KPKT_SC_BK:
		return 3;
	default:
		VERIFY(0);
		return 0;
	}
}

static errno_t
feth_nx_ring_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    kern_channel_t channel, kern_channel_ring_t ring, boolean_t is_tx_ring,
    void **ring_ctx)
{
	if_fake_ref     fakeif;
	int             err;
#pragma unused(nxprov, channel, ring_ctx)
	feth_lock();
	fakeif = feth_nexus_context(nexus);
	if (feth_is_detaching(fakeif)) {
		feth_unlock();
		return 0;
	}
	if (is_tx_ring) {
		if (feth_in_wmm_mode(fakeif)) {
			kern_packet_svc_class_t svc_class;
			uint8_t ring_idx;

			err = kern_channel_get_service_class(ring, &svc_class);
			VERIFY(err == 0);
			ring_idx = feth_find_tx_ring_by_svc(svc_class);
			VERIFY(ring_idx < IFF_NUM_TX_RINGS_WMM_MODE);
			VERIFY(fakeif->iff_tx_ring[ring_idx] == NULL);
			fakeif->iff_tx_ring[ring_idx] = ring;
		} else {
			VERIFY(fakeif->iff_tx_ring[0] == NULL);
			fakeif->iff_tx_ring[0] = ring;
		}
	} else {
		VERIFY(fakeif->iff_rx_ring[0] == NULL);
		fakeif->iff_rx_ring[0] = ring;
	}
	fakeif->iff_nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
	feth_unlock();
	FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: %s ring init",
	    fakeif->iff_name, is_tx_ring ? "TX" : "RX");
	return 0;
}

static void
feth_nx_ring_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    kern_channel_ring_t ring)
{
#pragma unused(nxprov, ring)
	if_fake_ref       fakeif;
	thread_call_t   __single tcall = NULL;

	feth_lock();
	fakeif = feth_nexus_context(nexus);
	if (fakeif->iff_rx_ring[0] == ring) {
		fakeif->iff_rx_ring[0] = NULL;
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "%s: RX ring fini", fakeif->iff_name);
	} else if (feth_in_wmm_mode(fakeif)) {
		int i;
		for (i = 0; i < IFF_MAX_TX_RINGS; i++) {
			if (fakeif->iff_tx_ring[i] == ring) {
				fakeif->iff_tx_ring[i] = NULL;
				break;
			}
		}
		for (i = 0; i < IFF_MAX_TX_RINGS; i++) {
			if (fakeif->iff_tx_ring[i] != NULL) {
				break;
			}
		}
		if (i == IFF_MAX_TX_RINGS) {
			tcall = fakeif->iff_doorbell_tcall;
			fakeif->iff_doorbell_tcall = NULL;
		}
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "%s: TX ring fini", fakeif->iff_name);
	} else if (fakeif->iff_tx_ring[0] == ring) {
		tcall = fakeif->iff_doorbell_tcall;
		fakeif->iff_doorbell_tcall = NULL;
		fakeif->iff_tx_ring[0] = NULL;
	}
	fakeif->iff_nifs = NULL;
	feth_unlock();
	if (tcall != NULL) {
		boolean_t       success;

		success = thread_call_cancel_wait(tcall);
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "%s: thread_call_cancel %s", fakeif->iff_name,
		    success ? "SUCCESS" : "FAILURE");
		if (!success) {
			feth_lock();
			if (fakeif->iff_doorbell_tcall_active) {
				fakeif->iff_waiting_for_tcall = TRUE;
				FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
				    "%s: *waiting for threadcall",
				    fakeif->iff_name);
				do {
					msleep(fakeif, &feth_lck_mtx,
					    PZERO, "feth threadcall", 0);
				} while (fakeif->iff_doorbell_tcall_active);
				FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
				    "%s: ^threadcall done",
				    fakeif->iff_name);
				fakeif->iff_waiting_for_tcall = FALSE;
			}
			feth_unlock();
		}
		success = thread_call_free(tcall);
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "%s: thread_call_free %s",
		    fakeif->iff_name,
		    success ? "SUCCESS" : "FAILURE");
		feth_release(fakeif);
		VERIFY(success == TRUE);
	}
}

static errno_t
feth_nx_pre_connect(kern_nexus_provider_t nxprov,
    proc_t proc, kern_nexus_t nexus, nexus_port_t port, kern_channel_t channel,
    void **channel_context)
{
#pragma unused(nxprov, proc, nexus, port, channel, channel_context)
	return 0;
}

static errno_t
feth_nx_connected(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
	int err;
	if_fake_ref fakeif;

	fakeif = feth_nexus_context(nexus);
	feth_lock();
	if (feth_is_detaching(fakeif)) {
		feth_unlock();
		return EBUSY;
	}
	feth_retain(fakeif);
	fakeif->iff_channel_connected = TRUE;
	feth_unlock();
	if (feth_has_intf_advisory_configured(fakeif)) {
		err = feth_if_adv_tcall_create(fakeif);
		if (err != 0) {
			return err;
		}
	}
	FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: connected channel %p",
	    fakeif->iff_name, channel);
	return 0;
}

static void
feth_nx_pre_disconnect(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
	if_fake_ref fakeif;
	thread_call_t __single tcall;
	boolean_t connected;

	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
	    "%s: pre-disconnect channel %p",
	    fakeif->iff_name, channel);
	/* Quiesce the interface and flush any pending outbound packets. */
	if_down(fakeif->iff_ifp);
	feth_lock();
	connected = fakeif->iff_channel_connected;
	fakeif->iff_channel_connected = FALSE;
	tcall = fakeif->iff_if_adv_tcall;
	fakeif->iff_if_adv_tcall = NULL;
	feth_unlock();
	if (tcall != NULL) {
		(void) thread_call_cancel_wait(tcall);
		if (!thread_call_free(tcall)) {
			boolean_t freed;
			(void) thread_call_cancel_wait(tcall);
			freed = thread_call_free(tcall);
			VERIFY(freed);
		}
		/* release for the interface advisory thread call */
		feth_release(fakeif);
	}
	if (connected) {
		feth_release(fakeif);
	}
}

static void
feth_nx_disconnected(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_t channel)
{
#pragma unused(nxprov, channel)
	if_fake_ref fakeif;

	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE, "%s: disconnected channel %p",
	    fakeif->iff_name, channel);
}

static errno_t
feth_nx_slot_init(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_ring_t ring, kern_channel_slot_t slot,
    uint32_t slot_index, struct kern_slot_prop **slot_prop_addr,
    void **slot_context)
{
#pragma unused(nxprov, nexus, ring, slot, slot_index, slot_prop_addr, slot_context)
	return 0;
}

static void
feth_nx_slot_fini(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_ring_t ring, kern_channel_slot_t slot,
    uint32_t slot_index)
{
#pragma unused(nxprov, nexus, ring, slot, slot_index)
}

static errno_t
feth_nx_sync_tx(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_ring_t tx_ring, uint32_t flags)
{
#pragma unused(nxprov)
	if_fake_ref             fakeif;
	ifnet_t                 ifp;
	kern_channel_slot_t     last_tx_slot = NULL;
	ifnet_t                 peer_ifp;
	if_fake_ref             peer_fakeif = NULL;
	struct kern_channel_ring_stat_increment stats;
	kern_channel_slot_t     tx_slot;
	struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
	kern_packet_t           pkts[IFF_MAX_BATCH_SIZE];
	uint32_t                n_pkts = 0;

	memset(&stats, 0, sizeof(stats));

	STATS_INC(nifs, NETIF_STATS_TX_SYNC);
	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
	    "%s ring %d flags 0x%x", fakeif->iff_name,
	    tx_ring->ckr_ring_id, flags);
	(void)flags;
	feth_lock();
	if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
		feth_unlock();
		return 0;
	}
	ifp = fakeif->iff_ifp;
	peer_ifp = fakeif->iff_peer;
	if (peer_ifp != NULL) {
		peer_fakeif = ifnet_get_if_fake(peer_ifp);
		if (peer_fakeif != NULL) {
			if (feth_is_detaching(peer_fakeif)) {
				FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
				    "%s peer fakeif %s is detaching",
				    fakeif->iff_name, peer_fakeif->iff_name);
				goto done;
			}
			if (!peer_fakeif->iff_channel_connected) {
				if (fakeif->iff_tx_exp_policy ==
				    IFF_TX_EXP_POLICY_DISABLED) {
					FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
					    "%s peer fakeif %s channel not connected, expn: %d",
					    fakeif->iff_name, peer_fakeif->iff_name,
					    fakeif->iff_tx_exp_policy);
					goto done;
				}
			}
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s no peer fakeif (peer %p)",
			    fakeif->iff_name, peer_ifp);
			goto done;
		}
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s no peer", fakeif->iff_name);
		goto done;
	}
	tx_slot = kern_channel_get_next_slot(tx_ring, NULL, NULL);
	while (tx_slot != NULL) {
		uint16_t off;
		kern_packet_t sph;

		/* detach the packet from the TX ring */
		sph = kern_channel_slot_get_packet(tx_ring, tx_slot);
		VERIFY(sph != 0);
		kern_channel_slot_detach_packet(tx_ring, tx_slot, sph);

		/* bpf tap output */
		off = kern_packet_get_headroom(sph);
		VERIFY(off >= fakeif->iff_tx_headroom);
		kern_packet_set_link_header_length(sph, ETHER_HDR_LEN);
		feth_packet_set_trace_tag(sph, IFF_TT_OUTPUT);
		bpf_tap_packet_out(ifp, DLT_EN10MB, sph, NULL, 0);

		/* drop packets, if requested */
		fakeif->iff_tx_pkts_count++;
		if (feth_tx_expired_error(fakeif, sph) ||
		    feth_tx_complete_error(fakeif, sph) ||
		    !peer_fakeif->iff_channel_connected) {
			fakeif->iff_tx_pkts_count = 0;
			kern_pbufpool_free(fakeif->iff_tx_pp, sph);
			STATS_INC(nifs, NETIF_STATS_DROP);
			goto next_tx_slot;
		}

		ASSERT(sph != 0);
		STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
		STATS_INC(nifs, NETIF_STATS_TX_PACKETS);

		stats.kcrsi_slots_transferred++;
		stats.kcrsi_bytes_transferred
		        += kern_packet_get_data_length(sph);

		/* prepare batch for receiver */
		pkts[n_pkts++] = sph;
		if (n_pkts == IFF_MAX_BATCH_SIZE) {
			feth_rx_submit(fakeif, peer_fakeif, pkts, n_pkts);
			feth_tx_complete(fakeif, pkts, n_pkts);
			n_pkts = 0;
		}

next_tx_slot:
		last_tx_slot = tx_slot;
		tx_slot = kern_channel_get_next_slot(tx_ring, tx_slot, NULL);
	}

	/* catch last batch for receiver */
	if (n_pkts != 0) {
		feth_rx_submit(fakeif, peer_fakeif, pkts, n_pkts);
		feth_tx_complete(fakeif, pkts, n_pkts);
		n_pkts = 0;
	}

	if (last_tx_slot != NULL) {
		kern_channel_advance_slot(tx_ring, last_tx_slot);
		kern_channel_increment_ring_net_stats(tx_ring, ifp, &stats);
	}
done:
	feth_unlock();
	return 0;
}

static errno_t
feth_nx_sync_rx(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_ring_t ring, uint32_t flags)
{
#pragma unused(nxprov, ring, flags)
	if_fake_ref           fakeif;
	struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;

	STATS_INC(nifs, NETIF_STATS_RX_SYNC);
	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_INPUT, "%s", fakeif->iff_name);
	return 0;
}

static errno_t
feth_nx_tx_dequeue_driver_managed(if_fake_ref fakeif, boolean_t doorbell_ctxt)
{
	int i;
	errno_t error = 0;
	boolean_t more;

	for (i = 0; i < IFF_NUM_TX_RINGS_WMM_MODE; i++) {
		kern_channel_ring_t ring = fakeif->iff_tx_ring[i];
		if (ring != NULL) {
			error = kern_channel_tx_refill(ring, UINT32_MAX,
			    UINT32_MAX, doorbell_ctxt, &more);
		}
		if (error != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s: TX refill ring %d (%s) %d",
			    fakeif->iff_name, ring->ckr_ring_id,
			    doorbell_ctxt ? "sync" : "async", error);
			if (!((error == EAGAIN) || (error == EBUSY))) {
				break;
			}
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s: TX refilled ring %d (%s)",
			    fakeif->iff_name, ring->ckr_ring_id,
			    doorbell_ctxt ? "sync" : "async");
		}
	}
	return error;
}

static void
feth_async_doorbell(thread_call_param_t arg0, thread_call_param_t arg1)
{
#pragma unused(arg1)
	errno_t                 error;
	if_fake_ref             fakeif = (if_fake_ref)arg0;
	kern_channel_ring_t     ring;
	boolean_t               more;

	feth_lock();
	ring = fakeif->iff_tx_ring[0];
	if (feth_is_detaching(fakeif) ||
	    !fakeif->iff_channel_connected ||
	    ring == NULL) {
		goto done;
	}
	fakeif->iff_doorbell_tcall_active = TRUE;
	feth_unlock();
	if (feth_in_wmm_mode(fakeif)) {
		error = feth_nx_tx_dequeue_driver_managed(fakeif, FALSE);
	} else {
		error = kern_channel_tx_refill(ring, UINT32_MAX,
		    UINT32_MAX, FALSE, &more);
	}
	if (error != 0) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s: TX refill failed %d",
		    fakeif->iff_name, error);
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s: TX refilled",
		    fakeif->iff_name);
	}

	feth_lock();
done:
	fakeif->iff_doorbell_tcall_active = FALSE;
	if (fakeif->iff_waiting_for_tcall) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s: threadcall waking up waiter", fakeif->iff_name);
		wakeup((caddr_t)fakeif);
	}
	feth_unlock();
}

static void
feth_schedule_async_doorbell(if_fake_ref fakeif)
{
	thread_call_t  __single tcall;

	feth_lock();
	if (feth_is_detaching(fakeif) || !fakeif->iff_channel_connected) {
		feth_unlock();
		return;
	}
	tcall = fakeif->iff_doorbell_tcall;
	if (tcall != NULL) {
		thread_call_enter(tcall);
	} else {
		tcall = thread_call_allocate_with_options(feth_async_doorbell,
		    (thread_call_param_t)fakeif,
		    THREAD_CALL_PRIORITY_KERNEL,
		    THREAD_CALL_OPTIONS_ONCE);
		if (tcall == NULL) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT,
			    "%s tcall alloc failed", fakeif->iff_name);
		} else {
			fakeif->iff_doorbell_tcall = tcall;
			feth_retain(fakeif);
			thread_call_enter(tcall);
		}
	}
	feth_unlock();
}

static errno_t
feth_nx_tx_doorbell(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, kern_channel_ring_t ring, uint32_t flags)
{
#pragma unused(nxprov, ring, flags)
	errno_t         error;
	if_fake_ref     fakeif;

	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s", fakeif->iff_name);

	if ((flags & KERN_NEXUS_TXDOORBELLF_ASYNC_REFILL) == 0) {
		boolean_t       more;
		/* synchronous tx refill */
		if (feth_in_wmm_mode(fakeif)) {
			error = feth_nx_tx_dequeue_driver_managed(fakeif, TRUE);
		} else {
			error = kern_channel_tx_refill(ring, UINT32_MAX,
			    UINT32_MAX, TRUE, &more);
		}
		if (error != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s: TX refill (sync) %d", fakeif->iff_name, error);
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s: TX refilled (sync)", fakeif->iff_name);
		}
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s: schedule async refill", fakeif->iff_name);
		feth_schedule_async_doorbell(fakeif);
	}
	return 0;
}

static errno_t
feth_netif_prepare(kern_nexus_t nexus, ifnet_t ifp)
{
	if_fake_ref fakeif;

	fakeif = (if_fake_ref)kern_nexus_get_context(nexus);
	feth_ifnet_set_attrs(fakeif, ifp);
	return 0;
}

static errno_t
feth_nx_intf_adv_config(void *prov_ctx, bool enable)
{
	if_fake_ref fakeif = prov_ctx;

	feth_lock();
	fakeif->iff_intf_adv_enabled = enable;
	feth_unlock();
	FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
	    "%s enable %d", fakeif->iff_name, enable);
	return 0;
}

static errno_t
fill_capab_interface_advisory(if_fake_ref fakeif, void *contents, uint32_t *len)
{
	struct kern_nexus_capab_interface_advisory * __single capab = contents;

	if (*len != sizeof(*capab)) {
		return EINVAL;
	}
	if (capab->kncia_version !=
	    KERN_NEXUS_CAPAB_INTERFACE_ADVISORY_VERSION_1) {
		return EINVAL;
	}
	if (!feth_has_intf_advisory_configured(fakeif)) {
		return ENOTSUP;
	}
	VERIFY(capab->kncia_notify != NULL);
	fakeif->iff_intf_adv_kern_ctx = capab->kncia_kern_context;
	fakeif->iff_intf_adv_notify = capab->kncia_notify;
	capab->kncia_provider_context = fakeif;
	capab->kncia_config = feth_nx_intf_adv_config;
	return 0;
}

static errno_t
feth_notify_steering_info(void *prov_ctx, void *qset_ctx,
    struct ifnet_traffic_descriptor_common *td, bool add)
{
#pragma unused(td)
	if_fake_ref fakeif = prov_ctx;
	fake_qset * __single qset = qset_ctx;

	FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
	    "%s: notify_steering_info: qset_id 0x%llx, %s",
	    fakeif->iff_name, qset->fqs_id, add ? "add" : "remove");
	return 0;
}

static errno_t
fill_capab_qset_extensions(if_fake_ref fakeif, void *contents, uint32_t *len)
{
	struct kern_nexus_capab_qset_extensions * __single capab = contents;

	if (*len != sizeof(*capab)) {
		return EINVAL;
	}
	if (capab->cqe_version !=
	    KERN_NEXUS_CAPAB_QSET_EXTENSIONS_VERSION_1) {
		return EINVAL;
	}
	capab->cqe_prov_ctx = fakeif;
	capab->cqe_notify_steering_info = feth_notify_steering_info;
	return 0;
}

static errno_t
feth_nx_capab_config(kern_nexus_provider_t nxprov, kern_nexus_t nx,
    kern_nexus_capab_t capab, void *contents, uint32_t *len)
{
#pragma unused(nxprov)
	errno_t error;
	if_fake_ref fakeif;

	fakeif = feth_nexus_context(nx);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL, "%s", fakeif->iff_name);

	switch (capab) {
	case KERN_NEXUS_CAPAB_INTERFACE_ADVISORY:
		error = fill_capab_interface_advisory(fakeif, contents, len);
		break;
	case KERN_NEXUS_CAPAB_QSET_EXTENSIONS:
		error = fill_capab_qset_extensions(fakeif, contents, len);
		break;
	default:
		error = ENOTSUP;
		break;
	}
	return error;
}

static int
feth_set_tso_mtu(ifnet_t ifp, uint32_t tso_v4_mtu, uint32_t tso_v6_mtu)
{
	int     error;

	error = ifnet_set_tso_mtu(ifp, AF_INET, tso_v4_mtu);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
		    "set TSO MTU IPv4 failed on %s, err %d",
		    if_name(ifp), error);
		return error;
	}
	error = ifnet_set_tso_mtu(ifp, AF_INET6, tso_v6_mtu);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
		    "set TSO MTU IPv6 failed on %s, err %d",
		    if_name(ifp), error);
		return error;
	}
	return 0;
}

static int
feth_set_tso_offload(ifnet_t ifp)
{
	ifnet_offload_t offload;
	int error;

	offload = IFNET_TSO_IPV4 | IFNET_TSO_IPV6;
	error = ifnet_set_offload(ifp, offload);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_CONTROL,
		    "set TSO offload failed on %s, err %d",
		    if_name(ifp), error);
		goto done;
	}
	error = feth_set_tso_mtu(ifp, if_fake_tso_buffer_size,
	    if_fake_tso_buffer_size);
done:
	return error;
}

static errno_t
create_netif_provider_and_instance(if_fake_ref fakeif,
    struct ifnet_init_eparams * init_params, ifnet_t *ifp,
    uuid_t * provider, uuid_t * instance)
{
	errno_t                 err;
	nexus_controller_t      controller = kern_nexus_shared_controller();
	struct kern_nexus_net_init net_init;
	nexus_name_t            provider_name;
	nexus_attr_t            __single nexus_attr = NULL;
	struct kern_nexus_provider_init prov_init = {
		.nxpi_version = KERN_NEXUS_DOMAIN_PROVIDER_CURRENT_VERSION,
		.nxpi_flags = NXPIF_VIRTUAL_DEVICE,
		.nxpi_pre_connect = feth_nx_pre_connect,
		.nxpi_connected = feth_nx_connected,
		.nxpi_pre_disconnect = feth_nx_pre_disconnect,
		.nxpi_disconnected = feth_nx_disconnected,
		.nxpi_ring_init = feth_nx_ring_init,
		.nxpi_ring_fini = feth_nx_ring_fini,
		.nxpi_slot_init = feth_nx_slot_init,
		.nxpi_slot_fini = feth_nx_slot_fini,
		.nxpi_sync_tx = feth_nx_sync_tx,
		.nxpi_sync_rx = feth_nx_sync_rx,
		.nxpi_tx_doorbell = feth_nx_tx_doorbell,
		.nxpi_config_capab = feth_nx_capab_config,
	};

	_CASSERT(IFF_MAX_RX_RINGS == 1);
	err = kern_nexus_attr_create(&nexus_attr);
	if (err != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "nexus attribute creation failed, error %d", err);
		goto failed;
	}
	if (feth_in_wmm_mode(fakeif)) {
		err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_TX_RINGS,
		    IFF_NUM_TX_RINGS_WMM_MODE);
		VERIFY(err == 0);
		err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_RX_RINGS,
		    IFF_NUM_RX_RINGS_WMM_MODE);
		VERIFY(err == 0);
		err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_QMAP,
		    NEXUS_QMAP_TYPE_WMM);
		VERIFY(err == 0);
	}

	err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_ANONYMOUS, 1);
	VERIFY(err == 0);
	snprintf((char *)provider_name, sizeof(provider_name),
	    "com.apple.netif.%s", fakeif->iff_name);
	err = kern_nexus_controller_register_provider(controller,
	    feth_nx_dom_prov,
	    provider_name,
	    &prov_init,
	    sizeof(prov_init),
	    nexus_attr,
	    provider);
	if (err != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "register provider failed, error %d", err);
		goto failed;
	}
	bzero(&net_init, sizeof(net_init));
	net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
	net_init.nxneti_flags = 0;
	net_init.nxneti_eparams = init_params;
	net_init.nxneti_lladdr = NULL;
	net_init.nxneti_prepare = feth_netif_prepare;
	net_init.nxneti_rx_pbufpool = fakeif->iff_rx_pp;
	net_init.nxneti_tx_pbufpool = fakeif->iff_tx_pp;
	err = kern_nexus_controller_alloc_net_provider_instance(controller,
	    *provider,
	    fakeif,
	    NULL,
	    instance,
	    &net_init,
	    ifp);
	if (err != 0) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "alloc_net_provider_instance failed, %d", err);
		kern_nexus_controller_deregister_provider(controller,
		    *provider);
		uuid_clear(*provider);
		goto failed;
	}
	if (feth_supports_tso(fakeif)) {
		if ((err = feth_set_tso_offload(*ifp)) != 0) {
			goto failed;
		}
	}

failed:
	if (nexus_attr != NULL) {
		kern_nexus_attr_destroy(nexus_attr);
	}
	return err;
}

/*
 * The nif_stats need to be referenced because we don't want it set
 * to NULL until the last llink is removed.
 */
static void
get_nexus_stats(if_fake_ref fakeif, kern_nexus_t nexus)
{
	if (++fakeif->iff_nifs_ref == 1) {
		ASSERT(fakeif->iff_nifs == NULL);
		fakeif->iff_nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
	}
}

static void
clear_nexus_stats(if_fake_ref fakeif)
{
	if (--fakeif->iff_nifs_ref == 0) {
		ASSERT(fakeif->iff_nifs != NULL);
		fakeif->iff_nifs = NULL;
	}
}

static errno_t
feth_nx_qset_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    void *llink_ctx, uint8_t qset_idx, uint64_t qset_id, kern_netif_qset_t qset,
    void **qset_ctx)
{
#pragma unused(nxprov)
	if_fake_ref fakeif;
	fake_llink * __single fl = llink_ctx;
	fake_qset *fqs;

	feth_lock();
	fakeif = feth_nexus_context(nexus);
	if (feth_is_detaching(fakeif)) {
		feth_unlock();
		FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
		    "%s: detaching", fakeif->iff_name);
		return ENXIO;
	}
	if (qset_idx >= fl->fl_qset_cnt) {
		feth_unlock();
		FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
		    "%s: invalid qset_idx %d", fakeif->iff_name, qset_idx);
		return EINVAL;
	}
	fqs = &fl->fl_qset[qset_idx];
	ASSERT(fqs->fqs_qset == NULL);
	fqs->fqs_qset = qset;
	fqs->fqs_id = qset_id;
	*qset_ctx = fqs;

	/* XXX This should really be done during registration */
	get_nexus_stats(fakeif, nexus);
	feth_unlock();
	return 0;
}

static void
feth_nx_qset_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    void *qset_ctx)
{
#pragma unused(nxprov)
	if_fake_ref fakeif;
	fake_qset * __single fqs = qset_ctx;

	feth_lock();
	fakeif = feth_nexus_context(nexus);
	clear_nexus_stats(fakeif);
	ASSERT(fqs->fqs_qset != NULL);
	fqs->fqs_qset = NULL;
	fqs->fqs_id = 0;
	feth_unlock();
}

static errno_t
feth_nx_queue_init(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    void *qset_ctx, uint8_t qidx, bool tx, kern_netif_queue_t queue,
    void **queue_ctx)
{
#pragma unused(nxprov)
	if_fake_ref fakeif;
	fake_qset *__single fqs = qset_ctx;
	fake_queue *fq;

	feth_lock();
	fakeif = feth_nexus_context(nexus);
	if (feth_is_detaching(fakeif)) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
		    "%s: detaching", fakeif->iff_name);
		feth_unlock();
		return ENXIO;
	}
	if (tx) {
		if (qidx >= fqs->fqs_tx_queue_cnt) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
			    "%s: invalid tx qidx %d", fakeif->iff_name, qidx);
			feth_unlock();
			return EINVAL;
		}
		fq = &fqs->fqs_tx_queue[qidx];
	} else {
		if (qidx >= fqs->fqs_rx_queue_cnt) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_CONTROL,
			    "%s: invalid rx qidx %d", fakeif->iff_name, qidx);
			feth_unlock();
			return EINVAL;
		}
		fq = &fqs->fqs_rx_queue[qidx];
	}
	ASSERT(fq->fq_queue == NULL);
	fq->fq_queue = queue;
	*queue_ctx = fq;
	feth_unlock();
	return 0;
}

static void
feth_nx_queue_fini(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    void *queue_ctx)
{
#pragma unused(nxprov, nexus)
	fake_queue *__single fq = queue_ctx;

	feth_lock();
	ASSERT(fq->fq_queue != NULL);
	fq->fq_queue = NULL;
	feth_unlock();
}

static void
feth_nx_tx_queue_deliver_pkt_chain(if_fake_ref fakeif, kern_packet_t sph,
    struct netif_stats *nifs, if_fake_ref peer_fakeif,
    uint32_t llink_idx, uint32_t qset_idx)
{
	kern_packet_t pkts[IFF_MAX_BATCH_SIZE];
	uint32_t n_pkts = 0;

	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
	    "%s -> %s", fakeif->iff_name, peer_fakeif->iff_name);

	while (sph != 0) {
		uint16_t off;
		kern_packet_t next;

		next = kern_packet_get_next(sph);
		kern_packet_set_next(sph, 0);

		/* bpf tap output */
		off = kern_packet_get_headroom(sph);
		VERIFY(off >= fakeif->iff_tx_headroom);
		kern_packet_set_link_header_length(sph, ETHER_HDR_LEN);
		feth_packet_set_trace_tag(sph, IFF_TT_OUTPUT);
		bpf_tap_packet_out(fakeif->iff_ifp, DLT_EN10MB, sph, NULL, 0);

		/* drop packets, if requested */
		fakeif->iff_tx_pkts_count++;
		if (feth_tx_expired_error(fakeif, sph) ||
		    feth_tx_complete_error(fakeif, sph)) {
			fakeif->iff_tx_pkts_count = 0;
			kern_pbufpool_free(fakeif->iff_tx_pp, sph);
			STATS_INC(nifs, NETIF_STATS_DROP);
			goto next_pkt;
		}
		ASSERT(sph != 0);
		STATS_INC(nifs, NETIF_STATS_TX_COPY_DIRECT);
		STATS_INC(nifs, NETIF_STATS_TX_PACKETS);

		/* prepare batch for receiver */
		pkts[n_pkts++] = sph;
		if (n_pkts == IFF_MAX_BATCH_SIZE) {
			feth_rx_queue_submit(fakeif, peer_fakeif, llink_idx,
			    qset_idx, pkts, n_pkts);
			feth_tx_complete(fakeif, pkts, n_pkts);
			n_pkts = 0;
		}
next_pkt:
		sph = next;
	}
	/* catch last batch for receiver */
	if (n_pkts != 0) {
		feth_rx_queue_submit(fakeif, peer_fakeif, llink_idx, qset_idx,
		    pkts, n_pkts);
		feth_tx_complete(fakeif, pkts, n_pkts);
		n_pkts = 0;
	}
}

static errno_t
feth_nx_tx_qset_notify(kern_nexus_provider_t nxprov, kern_nexus_t nexus,
    void *qset_ctx, uint32_t flags)
{
#pragma unused(nxprov)
	if_fake_ref             fakeif;
	ifnet_t                 ifp;
	ifnet_t                 peer_ifp;
	if_fake_ref             peer_fakeif = NULL;
	struct netif_stats *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
	fake_qset               * __single qset = qset_ctx;
	boolean_t               detaching, connected;
	uint32_t                i;
	errno_t                 err;

	STATS_INC(nifs, NETIF_STATS_TX_SYNC);
	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
	    "%s qset %p, idx %d, flags 0x%x", fakeif->iff_name, qset,
	    qset->fqs_idx, flags);

	feth_lock();
	detaching = feth_is_detaching(fakeif);
	connected = fakeif->iff_channel_connected;
	if (detaching || !connected) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s: detaching %s, channel connected %s",
		    fakeif->iff_name,
		    (detaching ? "true" : "false"),
		    (connected ? "true" : "false"));
		feth_unlock();
		return 0;
	}
	ifp = fakeif->iff_ifp;
	peer_ifp = fakeif->iff_peer;
	if (peer_ifp != NULL) {
		peer_fakeif = ifnet_get_if_fake(peer_ifp);
		if (peer_fakeif != NULL) {
			detaching = feth_is_detaching(peer_fakeif);
			connected = peer_fakeif->iff_channel_connected;
			if (detaching || !connected) {
				FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
				    "peer %s: detaching %s, "
				    "channel connected %s",
				    peer_fakeif->iff_name,
				    (detaching ? "true" : "false"),
				    (connected ? "true" : "false"));
				goto done;
			}
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "peer_fakeif is NULL");
			goto done;
		}
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "peer_ifp is NULL");
		goto done;
	}

	if (if_fake_switch_combined_mode &&
	    qset->fqs_dequeue_cnt >= if_fake_switch_mode_frequency) {
		if (qset->fqs_combined_mode) {
			kern_netif_set_qset_separate(qset->fqs_qset);
		} else {
			kern_netif_set_qset_combined(qset->fqs_qset);
		}
		qset->fqs_combined_mode = !qset->fqs_combined_mode;
		qset->fqs_dequeue_cnt = 0;
	}

	for (i = 0; i < qset->fqs_tx_queue_cnt; i++) {
		kern_packet_t sph = 0;
		kern_netif_queue_t queue = qset->fqs_tx_queue[i].fq_queue;
		boolean_t more = FALSE;

		err = kern_netif_queue_tx_dequeue(queue, UINT32_MAX, UINT32_MAX,
		    &more, &sph);
		if (err != 0 && err != EAGAIN) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "%s queue %p dequeue failed: err "
			    "%d", fakeif->iff_name, queue, err);
		}
		feth_nx_tx_queue_deliver_pkt_chain(fakeif, sph, nifs,
		    peer_fakeif, qset->fqs_llink_idx, qset->fqs_idx);
	}

done:
	feth_unlock();
	return 0;
}


static errno_t
feth_nx_queue_tx_push(kern_nexus_provider_t nxprov,
    kern_nexus_t nexus, void *queue_ctx, kern_packet_t *ph,
    uint32_t *packetCount, uint32_t *byteCount)
{
#pragma unused(nxprov)
	if_fake_ref             fakeif;
	ifnet_t                 ifp;
	ifnet_t                 peer_ifp;
	if_fake_ref             peer_fakeif = NULL;
	struct netif_stats      *nifs = &NX_NETIF_PRIVATE(nexus)->nif_stats;
	fake_queue              *__single fq = queue_ctx;
	boolean_t               detaching, connected;

	STATS_INC(nifs, NETIF_STATS_TX_SYNC);
	fakeif = feth_nexus_context(nexus);
	FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "%s queue %p", fakeif->iff_name, fq);

	feth_lock();

	detaching = feth_is_detaching(fakeif);
	connected = fakeif->iff_channel_connected;
	if (detaching || !connected) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s: detaching %s, channel connected %s",
		    fakeif->iff_name,
		    (detaching ? "true" : "false"),
		    (connected ? "true" : "false"));
		goto done;
	}
	ifp = fakeif->iff_ifp;
	peer_ifp = fakeif->iff_peer;
	if (peer_ifp != NULL) {
		peer_fakeif = ifnet_get_if_fake(peer_ifp);
		if (peer_fakeif != NULL) {
			detaching = feth_is_detaching(peer_fakeif);
			connected = peer_fakeif->iff_channel_connected;
			if (detaching || !connected) {
				FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
				    "peer %s: detaching %s, "
				    "channel connected %s",
				    peer_fakeif->iff_name,
				    (detaching ? "true" : "false"),
				    (connected ? "true" : "false"));
				goto done;
			}
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
			    "peer_fakeif is NULL");
			goto done;
		}
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT, "peer_ifp is NULL");
		goto done;
	}

	*packetCount = *byteCount = 0;

	kern_packet_t sph = *ph;
	while (sph != 0) {
		(*packetCount)++;
		*byteCount += kern_packet_get_data_length(sph);
		sph = kern_packet_get_next(sph);
	}

	feth_nx_tx_queue_deliver_pkt_chain(fakeif, *ph, nifs,
	    peer_fakeif, 0, 0);

	*ph = 0;

done:
	feth_unlock();
	return 0;
}


static void
fill_qset_info_and_params(if_fake_ref fakeif, fake_llink *llink_info,
    uint32_t qset_idx, struct kern_nexus_netif_llink_qset_init *qset_init,
    bool is_def, bool is_low_latency)
{
	fake_qset *qset_info = &llink_info->fl_qset[qset_idx];

	qset_init->nlqi_flags =
	    (is_def ? KERN_NEXUS_NET_LLINK_QSET_DEFAULT : 0) |
	    (is_low_latency ? KERN_NEXUS_NET_LLINK_QSET_LOW_LATENCY : 0) |
	    KERN_NEXUS_NET_LLINK_QSET_AQM;

	if (feth_in_wmm_mode(fakeif)) {
		qset_init->nlqi_flags |= KERN_NEXUS_NET_LLINK_QSET_WMM_MODE;
		qset_init->nlqi_num_txqs = IFF_NUM_TX_QUEUES_WMM_MODE;
		qset_init->nlqi_num_rxqs = IFF_NUM_RX_QUEUES_WMM_MODE;
	} else {
		qset_init->nlqi_num_txqs = 1;
		qset_init->nlqi_num_rxqs = 1;
	}
	qset_info->fqs_tx_queue_cnt = qset_init->nlqi_num_txqs;
	qset_info->fqs_rx_queue_cnt = qset_init->nlqi_num_rxqs;

	/* These are needed for locating the peer qset */
	qset_info->fqs_llink_idx = llink_info->fl_idx;
	qset_info->fqs_idx = qset_idx;
}

static void
fill_llink_info_and_params(if_fake_ref fakeif, uint32_t llink_idx,
    struct kern_nexus_netif_llink_init *llink_init, uint32_t llink_id,
    struct kern_nexus_netif_llink_qset_init * __counted_by(qset_cnt) qset_init, uint32_t qset_cnt,
    uint32_t flags)
{
	fake_llink *llink_info = &fakeif->iff_llink[llink_idx];
	uint32_t i;
	bool create_ll_qset = if_fake_low_latency && (llink_idx != 0);

	for (i = 0; i < qset_cnt; i++) {
		fill_qset_info_and_params(fakeif, llink_info, i,
		    &qset_init[i], i == 0, create_ll_qset && i == 1);
	}
	llink_info->fl_idx = llink_idx;

	/* This doesn't have to be the same as llink_idx */
	llink_info->fl_id = llink_id;
	llink_info->fl_qset_cnt = qset_cnt;

	llink_init->nli_link_id = llink_id;
	llink_init->nli_num_qsets = qset_cnt;
	llink_init->nli_qsets = qset_init;
	llink_init->nli_flags = flags;
	llink_init->nli_ctx = llink_info;
}

static errno_t
create_non_default_llinks(if_fake_ref fakeif)
{
	struct kern_nexus *nx;
	fake_nx_t fnx = &fakeif->iff_nx;
	struct kern_nexus_netif_llink_init llink_init;
	struct kern_nexus_netif_llink_qset_init qset_init[FETH_MAX_QSETS];
	errno_t err;
	uint64_t llink_id;
	uint32_t i;

	nx = nx_find(fnx->fnx_instance, FALSE);
	if (nx == NULL) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
		    "%s: nx not found", fakeif->iff_name);
		return ENXIO;
	}
	/* Default llink starts at index 0 */
	for (i = 1; i < if_fake_llink_cnt; i++) {
		llink_id = (uint64_t)i;

		/*
		 * The llink_init and qset_init structures are reused for
		 * each llink creation.
		 */
		fill_llink_info_and_params(fakeif, i, &llink_init,
		    llink_id, qset_init, if_fake_qset_cnt, 0);
		err = kern_nexus_netif_llink_add(nx, &llink_init);
		if (err != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
			    "%s: llink add failed, error %d",
			    fakeif->iff_name, err);
			goto fail;
		}
		fakeif->iff_llink_cnt++;
	}
	nx_release(nx);
	return 0;

fail:
	for (i = 0; i < fakeif->iff_llink_cnt; i++) {
		int                     error;
		fake_llink * __single   ll = &fakeif->iff_llink[i];

		error = kern_nexus_netif_llink_remove(nx, ll->fl_id);
		if (error != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_MISC,
			    "%s: llink remove failed, llink_id 0x%llx, "
			    "error %d", fakeif->iff_name,
			    ll->fl_id, error);
		}
		ll->fl_id = 0;
	}
	fakeif->iff_llink_cnt = 0;
	nx_release(nx);
	return err;
}

static errno_t
create_netif_llink_provider_and_instance(if_fake_ref fakeif,
    struct ifnet_init_eparams * init_params, ifnet_t *ifp,
    uuid_t * provider, uuid_t * instance)
{
	errno_t                 err;
	nexus_controller_t      controller = kern_nexus_shared_controller();
	struct kern_nexus_net_init net_init;
	struct kern_nexus_netif_llink_init llink_init;
	struct kern_nexus_netif_llink_qset_init qsets[FETH_MAX_QSETS];

	nexus_name_t            provider_name;
	nexus_attr_t            __single nexus_attr = NULL;
	struct kern_nexus_netif_provider_init prov_init = {
		.nxnpi_version = KERN_NEXUS_DOMAIN_PROVIDER_NETIF,
		.nxnpi_flags = NXPIF_VIRTUAL_DEVICE,
		.nxnpi_pre_connect = feth_nx_pre_connect,
		.nxnpi_connected = feth_nx_connected,
		.nxnpi_pre_disconnect = feth_nx_pre_disconnect,
		.nxnpi_disconnected = feth_nx_disconnected,
		.nxnpi_qset_init = feth_nx_qset_init,
		.nxnpi_qset_fini = feth_nx_qset_fini,
		.nxnpi_queue_init = feth_nx_queue_init,
		.nxnpi_queue_fini = feth_nx_queue_fini,
		.nxnpi_tx_qset_notify = feth_nx_tx_qset_notify,
		.nxnpi_config_capab = feth_nx_capab_config,
		.nxnpi_queue_tx_push = feth_nx_queue_tx_push
	};

	err = kern_nexus_attr_create(&nexus_attr);
	if (err != 0) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "nexus attribute creation failed, error %d", err);
		goto failed;
	}

	err = kern_nexus_attr_set(nexus_attr, NEXUS_ATTR_ANONYMOUS, 1);
	VERIFY(err == 0);

	snprintf((char *)provider_name, sizeof(provider_name),
	    "com.apple.netif.%s", fakeif->iff_name);
	err = kern_nexus_controller_register_provider(controller,
	    feth_nx_dom_prov,
	    provider_name,
	    (struct kern_nexus_provider_init *)&prov_init,
	    sizeof(prov_init),
	    nexus_attr,
	    provider);
	if (err != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "register provider failed, error %d", err);
		goto failed;
	}
	bzero(&net_init, sizeof(net_init));
	net_init.nxneti_version = KERN_NEXUS_NET_CURRENT_VERSION;
	net_init.nxneti_flags = 0;
	net_init.nxneti_eparams = init_params;
	net_init.nxneti_lladdr = NULL;
	net_init.nxneti_prepare = feth_netif_prepare;
	net_init.nxneti_rx_pbufpool = fakeif->iff_rx_pp;
	net_init.nxneti_tx_pbufpool = fakeif->iff_tx_pp;

	/*
	 * Assume llink id is same as the index for if_fake.
	 * This is not required for other drivers.
	 */
	_CASSERT(NETIF_LLINK_ID_DEFAULT == 0);
	fill_llink_info_and_params(fakeif, 0, &llink_init,
	    NETIF_LLINK_ID_DEFAULT, qsets, if_fake_qset_cnt,
	    KERN_NEXUS_NET_LLINK_DEFAULT);

	net_init.nxneti_llink = &llink_init;

	err = kern_nexus_controller_alloc_net_provider_instance(controller,
	    *provider, fakeif, NULL, instance, &net_init, ifp);
	if (err != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "alloc_net_provider_instance failed, %d", err);
		kern_nexus_controller_deregister_provider(controller,
		    *provider);
		uuid_clear(*provider);
		goto failed;
	}
	fakeif->iff_llink_cnt++;

	if (if_fake_llink_cnt > 1) {
		err = create_non_default_llinks(fakeif);
		if (err != 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "create_non_default_llinks failed, %d", err);
			feth_detach_netif_nexus(fakeif);
			goto failed;
		}
	}
	if (feth_supports_tso(fakeif)) {
		if ((err = feth_set_tso_offload(*ifp)) != 0) {
			goto failed;
		}
	}
failed:
	if (nexus_attr != NULL) {
		kern_nexus_attr_destroy(nexus_attr);
	}
	return err;
}

static errno_t
feth_attach_netif_nexus(if_fake_ref fakeif,
    struct ifnet_init_eparams * init_params, ifnet_t *ifp)
{
	errno_t                 error;
	fake_nx_t               nx = &fakeif->iff_nx;

	error = feth_packet_pool_make(fakeif);
	if (error != 0) {
		return error;
	}
	if (if_fake_llink_cnt == 0) {
		return create_netif_provider_and_instance(fakeif, init_params,
		           ifp, &nx->fnx_provider, &nx->fnx_instance);
	} else {
		return create_netif_llink_provider_and_instance(fakeif,
		           init_params, ifp, &nx->fnx_provider,
		           &nx->fnx_instance);
	}
}

static void
remove_non_default_llinks(const char * name, fake_nx_t fnx,
    fake_llink_t llink __counted_by(FETH_MAX_LLINKS),
    uint32_t llink_cnt)
{
	struct kern_nexus *nx;
	uint32_t i;

	if (llink_cnt <= 1) {
		goto done;
	}
	nx = nx_find(fnx->fnx_instance, FALSE);
	if (nx == NULL) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "%s: nx not found", name);
		goto done;
	}
	/* Default llink (at index 0) is freed separately */
	for (i = 1; i < llink_cnt; i++) {
		int err;

		err = kern_nexus_netif_llink_remove(nx, llink[i].fl_id);
		if (err != 0) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
			    "%s: llink remove failed, llink_id 0x%llx, "
			    "error %d", name,
			    llink[i].fl_id, err);
		}
	}
	nx_release(nx);
done:
	return;
}

static void
detach_provider_and_instance(uuid_t provider, uuid_t instance)
{
	nexus_controller_t controller = kern_nexus_shared_controller();
	errno_t err;

	if (!uuid_is_null(instance)) {
		err = kern_nexus_controller_free_provider_instance(controller,
		    instance);
		if (err != 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "free_provider_instance failed %d", err);
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
			    "deregister_instance");
		}
	}
	if (!uuid_is_null(provider)) {
		err = kern_nexus_controller_deregister_provider(controller,
		    provider);
		if (err != 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "deregister_provider %d", err);
		} else {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
			    "deregister_provider");
		}
	}
	return;
}

static void
feth_detach_netif_nexus(if_fake_ref fakeif)
{
	fake_nx         fnx;
	fake_llink_t    llink;
	uint32_t        llink_cnt;

	feth_lock();
	fnx = fakeif->iff_nx;
	bzero(&fakeif->iff_nx, sizeof(fakeif->iff_nx));
	llink = fakeif->iff_llink;
	fakeif->iff_llink = NULL;
	llink_cnt = fakeif->iff_llink_cnt;
	fakeif->iff_llink_cnt = 0;
	feth_unlock();
	remove_non_default_llinks(__unsafe_null_terminated_from_indexable(fakeif->iff_name), &fnx, llink, llink_cnt);
	detach_provider_and_instance(fnx.fnx_provider, fnx.fnx_instance);
	if (llink != NULL) {
		kfree_type(fake_llink, FETH_MAX_LLINKS, llink);
	}
	return;
}
#endif /* SKYWALK */

/**
** feth interface routines
**/
static void
feth_ifnet_set_attrs(if_fake_ref fakeif, ifnet_t ifp)
{
	errno_t         error;
	ifnet_offload_t offload = 0;

	ifnet_set_addrlen(ifp, ETHER_ADDR_LEN);
	ifnet_set_baudrate(ifp, 0);
	ifnet_set_mtu(ifp, ETHERMTU);
	ifnet_set_flags(ifp,
	    IFF_BROADCAST | IFF_MULTICAST | IFF_SIMPLEX,
	    0xffff);
	ifnet_set_hdrlen(ifp, sizeof(struct ether_header));
	if ((fakeif->iff_flags & IFF_FLAGS_LRO) != 0) {
		offload |= IFNET_LRO;
	}
	if ((fakeif->iff_flags & IFF_FLAGS_HWCSUM) != 0) {
		offload |= IFNET_CSUM_IP | IFNET_CSUM_TCP | IFNET_CSUM_UDP |
		    IFNET_CSUM_TCPIPV6 | IFNET_CSUM_UDPIPV6;
	}
	if (feth_supports_tso(fakeif)) {
		offload |= IFNET_TSO_IPV4 | IFNET_TSO_IPV6;
	}
	if (feth_supports_vlan_tagging(fakeif)) {
		offload |= IFNET_VLAN_TAGGING;
	} else if (feth_supports_vlan_mtu(fakeif)) {
		offload |= IFNET_VLAN_MTU;
	}
	error = ifnet_set_offload(ifp, offload);
	if (error != 0) {
		FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
		    "ifnet_set_offload(%s, 0x%x) failed, %d",
		    ifp->if_xname, offload, error);
	} else {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
		    "ifnet_set_offload(%s, 0x%x) succeeded",
		    ifp->if_xname, offload);
	}
}

static void
interface_link_event(ifnet_t ifp, u_int32_t event_code)
{
	struct event {
		u_int32_t ifnet_family;
		u_int32_t unit;
		char if_name[IFNAMSIZ];
	};
	_Alignas(struct kern_event_msg) char message[sizeof(struct kern_event_msg) + sizeof(struct event)] = { 0 };
	struct kern_event_msg *__single header = (struct kern_event_msg*)message;
	struct event *data = (struct event *)(message + offsetof(struct kern_event_msg, event_data));

	header->total_size   = sizeof(message);
	header->vendor_code  = KEV_VENDOR_APPLE;
	header->kev_class    = KEV_NETWORK_CLASS;
	header->kev_subclass = KEV_DL_SUBCLASS;
	header->event_code   = event_code;
	data->ifnet_family   = ifnet_family(ifp);
	data->unit           = (u_int32_t)ifnet_unit(ifp);
	strlcpy(data->if_name, ifnet_name(ifp), IFNAMSIZ);
	ifnet_event(ifp, header);
}

static if_fake_ref
ifnet_get_if_fake(ifnet_t ifp)
{
	return (if_fake_ref)ifnet_softc(ifp);
}

static int
feth_clone_create(struct if_clone *ifc, u_int32_t unit, __unused void *params)
{
	bool                            bsd_mode;
	int                             error;
	if_fake_ref                   fakeif;
	struct ifnet_init_eparams       feth_init;
	fake_llink_t                    iff_llink __counted_by_or_null(FETH_MAX_LLINKS) = NULL;
	ifnet_t                         __single ifp;
	char                            mac_address[ETHER_ADDR_LEN];
	bool                            multi_buflet;
	iff_pktpool_mode_t              pktpool_mode;
	bool                            tso_support;

	/* make local copy of globals needed to make consistency checks below */
	bsd_mode = (if_fake_bsd_mode != 0);
	multi_buflet = (if_fake_multibuflet != 0);
	tso_support = (if_fake_tso_support != 0);
	pktpool_mode = if_fake_pktpool_mode;

	if (!bsd_mode) {
		/* consistency checks */
		if (if_fake_llink_cnt == 0 &&
		    strbufcmp(sk_ll_prefix, FAKE_ETHER_NAME) == 0) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_LIFECYCLE,
			    "feth used as ifname prefix but logical link "
			    "support in feth is disabled.");
			return EINVAL;
		}
		if (tso_support && pktpool_mode != IFF_PP_MODE_GLOBAL) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
			    "TSO mode requires global packet pool mode");
			return EINVAL;
		}
		if (multi_buflet && pktpool_mode == IFF_PP_MODE_PRIVATE_SPLIT) {
			FAKE_LOG(LOG_DEBUG, FE_DBGF_LIFECYCLE,
			    "multi-buflet not supported for split rx & tx pool");
			return EINVAL;
		}
		iff_llink = kalloc_type(fake_llink,
		    FETH_MAX_LLINKS, Z_WAITOK_ZERO);
		if (iff_llink == NULL) {
			return ENOBUFS;
		}
	}
	fakeif = kalloc_type(struct if_fake, Z_WAITOK_ZERO_NOFAIL);
	fakeif->iff_llink = iff_llink;
	fakeif->iff_retain_count = 1;
#define FAKE_ETHER_NAME_LEN     (sizeof(FAKE_ETHER_NAME) - 1)
	_CASSERT(FAKE_ETHER_NAME_LEN == 4);
	strbufcpy(mac_address, FAKE_ETHER_NAME);
	mac_address[ETHER_ADDR_LEN - 2] = (unit & 0xff00) >> 8;
	mac_address[ETHER_ADDR_LEN - 1] = unit & 0xff;
	if (bsd_mode) {
		fakeif->iff_flags |= IFF_FLAGS_BSD_MODE;
	}
	if (if_fake_hwcsum != 0) {
		fakeif->iff_flags |= IFF_FLAGS_HWCSUM;
	}
	if (if_fake_lro != 0) {
		fakeif->iff_flags |= IFF_FLAGS_LRO;
	}
	if (if_fake_vlan_tagging != 0) {
		/* support VLAN tagging in hardware */
		feth_set_supports_vlan_tagging(fakeif);
	} else {
		/* support VLAN mtu-sized packets */
		feth_set_supports_vlan_mtu(fakeif);
	}
	if (if_fake_separate_frame_header != 0) {
		fakeif->iff_flags |= IFF_FLAGS_SEPARATE_FRAME_HEADER;
	}
	fakeif->iff_max_mtu = get_max_mtu(bsd_mode, if_fake_max_mtu);
	fakeif->iff_fcs = if_fake_fcs;
	fakeif->iff_trailer_length = if_fake_trailer_length;

	/* use the interface name as the unique id for ifp recycle */
	if ((unsigned int)
	    snprintf(fakeif->iff_name, sizeof(fakeif->iff_name), "%s%d",
	    ifc->ifc_name, unit) >= sizeof(fakeif->iff_name)) {
		feth_release(fakeif);
		return EINVAL;
	}
	bzero(&feth_init, sizeof(feth_init));
	feth_init.ver = IFNET_INIT_CURRENT_VERSION;
	feth_init.len = sizeof(feth_init);
	if (feth_in_bsd_mode(fakeif)) {
		if (if_fake_txstart != 0) {
			feth_init.start = feth_start;
		} else {
			feth_init.flags |= IFNET_INIT_LEGACY;
			feth_init.output = feth_output;
		}
		if (tso_support) {
			feth_set_supports_tso(fakeif);
		}
	}
#if SKYWALK
	else {
		feth_init.flags |= IFNET_INIT_SKYWALK_NATIVE;
		/*
		 * Currently we support WMM mode only for Skywalk native
		 * interface.
		 */
		if (if_fake_wmm_mode != 0) {
			fakeif->iff_flags |= IFF_FLAGS_WMM_MODE;
		}

		if (multi_buflet) {
			fakeif->iff_flags |= IFF_FLAGS_MULTIBUFLETS;
		}

		fakeif->iff_pp_mode = pktpool_mode;
		if (tso_support) {
			feth_set_supports_tso(fakeif);
		}

		fakeif->iff_tx_headroom = if_fake_tx_headroom;
		fakeif->iff_adv_interval = if_fake_if_adv_interval;
		if (fakeif->iff_adv_interval > 0) {
			feth_init.flags |= IFNET_INIT_IF_ADV;
		}
		fakeif->iff_tx_drop_rate = if_fake_tx_drops;
		fakeif->iff_tx_completion_mode = if_tx_completion_mode;
		fakeif->iff_tx_exp_policy = if_fake_tx_exp_policy;
	}
	feth_init.tx_headroom = fakeif->iff_tx_headroom;
#endif /* SKYWALK */
	if (if_fake_nxattach == 0) {
		feth_init.flags |= IFNET_INIT_NX_NOAUTO;
	}
	feth_init.uniqueid_len = (uint32_t)strbuflen(fakeif->iff_name);
	feth_init.uniqueid = fakeif->iff_name;
	feth_init.name = __unsafe_null_terminated_from_indexable(ifc->ifc_name);
	feth_init.unit = unit;
	feth_init.family = IFNET_FAMILY_ETHERNET;
	feth_init.type = IFT_ETHER;
	feth_init.demux = ether_demux;
	feth_init.add_proto = ether_add_proto;
	feth_init.del_proto = ether_del_proto;
	feth_init.check_multi = ether_check_multi;
	feth_init.framer_extended = ether_frameout_extended;
	feth_init.softc = fakeif;
	feth_init.ioctl = feth_ioctl;
	feth_init.set_bpf_tap = NULL;
	feth_init.detach = feth_if_free;
	feth_init.broadcast_addr = etherbroadcastaddr;
	feth_init.broadcast_len = ETHER_ADDR_LEN;
	if (feth_in_bsd_mode(fakeif)) {
		error = ifnet_allocate_extended(&feth_init, &ifp);
		if (error) {
			feth_release(fakeif);
			return error;
		}
		feth_ifnet_set_attrs(fakeif, ifp);
		if (feth_supports_tso(fakeif)) {
			feth_set_tso_mtu(ifp, IP_MAXPACKET, IP_MAXPACKET);
		}
	}
#if SKYWALK
	else {
		if (feth_in_wmm_mode(fakeif)) {
			feth_init.output_sched_model =
			    IFNET_SCHED_MODEL_DRIVER_MANAGED;
		}
		error = feth_attach_netif_nexus(fakeif, &feth_init, &ifp);
		if (error != 0) {
			feth_release(fakeif);
			return error;
		}
		/* take an additional reference to ensure that it doesn't go away */
		feth_retain(fakeif);
		fakeif->iff_flags |= IFF_FLAGS_NX_ATTACHED;
		fakeif->iff_ifp = ifp;
	}
#endif /* SKYWALK */
	fakeif->iff_media_count = MIN(default_media_words_count, IF_FAKE_MEDIA_LIST_MAX);
	bcopy(default_media_words, fakeif->iff_media_list,
	    fakeif->iff_media_count * sizeof(fakeif->iff_media_list[0]));
	if (feth_in_bsd_mode(fakeif)) {
		error = ifnet_attach(ifp, NULL);
		if (error) {
			ifnet_release(ifp);
			feth_release(fakeif);
			return error;
		}
		fakeif->iff_ifp = ifp;
	}

	ifnet_set_lladdr(ifp, mac_address, sizeof(mac_address));

	/* attach as ethernet */
	bpfattach(ifp, DLT_EN10MB, sizeof(struct ether_header));
	return 0;
}

static int
feth_clone_destroy(ifnet_t ifp)
{
	if_fake_ref     fakeif;
#if SKYWALK
	boolean_t       nx_attached = FALSE;
#endif /* SKYWALK */

	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif == NULL || feth_is_detaching(fakeif)) {
		feth_unlock();
		return 0;
	}
	feth_set_detaching(fakeif);
#if SKYWALK
	nx_attached = (fakeif->iff_flags & IFF_FLAGS_NX_ATTACHED) != 0;
#endif /* SKYWALK */
	feth_unlock();
	feth_config(ifp, NULL);
#if SKYWALK
	if (nx_attached) {
		feth_detach_netif_nexus(fakeif);
		feth_release(fakeif);
	}
#endif /* SKYWALK */
	ifnet_detach(ifp);
	return 0;
}

static void
feth_enqueue_input(ifnet_t ifp, struct mbuf * m)
{
	struct ifnet_stat_increment_param stats = {};

	stats.packets_in = 1;
	stats.bytes_in = (uint32_t)mbuf_pkthdr_len(m) + ETHER_HDR_LEN;
	ifnet_input(ifp, m, &stats);
}


static int
feth_add_mbuf_trailer(struct mbuf *m, void *trailer __sized_by(trailer_len), size_t trailer_len)
{
	int ret;
	ASSERT(trailer_len <= FETH_TRAILER_LENGTH_MAX);

	ret = m_append(m, trailer_len, (caddr_t)trailer);
	if (ret == 1) {
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%zuB trailer added", trailer_len);
		return 0;
	}
	FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT, "m_append failed");
	return ENOTSUP;
}

static int
feth_add_mbuf_fcs(struct mbuf *m)
{
	uint32_t pkt_len, offset = 0;
	uint32_t crc = 0;
	int err = 0;

	ASSERT(sizeof(crc) == ETHER_CRC_LEN);

	pkt_len = m->m_pkthdr.len;
	struct mbuf *iter = m;
	while (iter != NULL && offset < pkt_len) {
		uint32_t frag_len = iter->m_len;
		ASSERT(frag_len <= (pkt_len - offset));
		crc = crc32(crc, mtod(iter, void *), frag_len);
		offset += frag_len;
		iter = iter->m_next;
	}

	err = feth_add_mbuf_trailer(m, &crc, ETHER_CRC_LEN);
	if (err != 0) {
		return err;
	}

	m->m_flags |= M_HASFCS;

	return 0;
}

static void
feth_output_common(ifnet_t ifp, struct mbuf * m, ifnet_t peer,
    iff_flags_t flags, bool fcs, void *trailer __sized_by(trailer_len), size_t trailer_len)
{
	void *                  frame_header;

	if ((flags & IFF_FLAGS_HWCSUM) != 0) {
		m->m_pkthdr.csum_data = 0xffff;
		m->m_pkthdr.csum_flags =
		    CSUM_DATA_VALID | CSUM_PSEUDO_HDR |
		    CSUM_IP_CHECKED | CSUM_IP_VALID;
	}

	(void)ifnet_stat_increment_out(ifp, 1, m->m_pkthdr.len, 0);
	if (trailer_len != 0 && trailer != NULL) {
		feth_add_mbuf_trailer(m, trailer, trailer_len);
	}
	if (fcs) {
		feth_add_mbuf_fcs(m);
	}
	if ((flags & IFF_FLAGS_SEPARATE_FRAME_HEADER) != 0) {
		m = m_copyup(m, ETHER_HDR_LEN, 0);
		if (m == NULL) {
			FAKE_LOG(LOG_NOTICE, FE_DBGF_OUTPUT, "m_copyup failed");
			goto done;
		}
		frame_header = mtod(m, void *);
		mbuf_pkthdr_setheader(m, frame_header);
		m_adj(m, ETHER_HDR_LEN);
		FAKE_LOG(LOG_DEBUG, FE_DBGF_OUTPUT,
		    "%s: frame 0x%llx data 0x%llx len %ld",
		    ifp->if_xname,
		    (uint64_t)VM_KERNEL_ADDRPERM(frame_header),
		    (uint64_t)VM_KERNEL_ADDRPERM(mtod(m, void *)),
		    mbuf_len(m));
	} else {
		frame_header = mtod(m, void *);
		mbuf_pkthdr_setheader(m, frame_header);
		_mbuf_adjust_pkthdr_and_data(m, ETHER_HDR_LEN);
	}

	/* tap it out */
	if (ifp->if_bpf != NULL) {
		fake_bpf_tap_out(ifp, m, frame_header);
	}

	/* tap it in */
	if (peer->if_bpf != NULL) {
		fake_bpf_tap_in(peer, m, frame_header);
	}

	(void)mbuf_pkthdr_setrcvif(m, peer);
	feth_enqueue_input(peer, m);
done:
	return;
}

static void
feth_start(ifnet_t ifp)
{
	if_fake_ref     fakeif;
	iff_flags_t     flags = 0;
	bool            fcs;
	struct mbuf *   __single m;
	ifnet_t         peer = NULL;
	size_t          trailer_len;

	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif == NULL) {
		feth_unlock();
		return;
	}

	if (fakeif->iff_start_busy) {
		feth_unlock();
		return;
	}

	peer = fakeif->iff_peer;
	flags = fakeif->iff_flags;
	fcs = fakeif->iff_fcs;
	trailer_len = fakeif->iff_trailer_length;

	fakeif->iff_start_busy = TRUE;
	feth_unlock();
	for (;;) {
		if (ifnet_dequeue(ifp, &m) != 0) {
			break;
		}
		if (peer == NULL) {
			m_freem(m);
			continue;
		}
		if (m != NULL) {
			feth_output_common(ifp, m, peer, flags, fcs,
			    feth_trailer, trailer_len);
		}
	}
	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif != NULL) {
		fakeif->iff_start_busy = FALSE;
	}
	feth_unlock();
}

static int
feth_output(ifnet_t ifp, struct mbuf * m)
{
	if_fake_ref             fakeif;
	iff_flags_t             flags;
	bool                    fcs;
	size_t                  trailer_len;
	ifnet_t                 peer = NULL;

	if (m == NULL) {
		return 0;
	}
	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif != NULL) {
		peer = fakeif->iff_peer;
		flags = fakeif->iff_flags;
		fcs = fakeif->iff_fcs;
		trailer_len = fakeif->iff_trailer_length;
	}
	feth_unlock();
	if (peer == NULL) {
		m_freem(m);
		ifnet_stat_increment_out(ifp, 0, 0, 1);
		return 0;
	}
	feth_output_common(ifp, m, peer, flags, fcs, feth_trailer, trailer_len);
	return 0;
}

static int
feth_config(ifnet_t ifp, ifnet_t peer)
{
	int             connected = FALSE;
	int             disconnected = FALSE;
	int             error = 0;
	if_fake_ref     fakeif = NULL;

	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif == NULL) {
		error = EINVAL;
		goto done;
	}
	if (peer != NULL) {
		/* connect to peer */
		if_fake_ref   peer_fakeif;

		peer_fakeif = ifnet_get_if_fake(peer);
		if (peer_fakeif == NULL) {
			error = EINVAL;
			goto done;
		}
		if (feth_is_detaching(fakeif) ||
		    feth_is_detaching(peer_fakeif) ||
		    peer_fakeif->iff_peer != NULL ||
		    fakeif->iff_peer != NULL) {
			error = EBUSY;
			goto done;
		}
#if SKYWALK
		if (fakeif->iff_pp_mode !=
		    peer_fakeif->iff_pp_mode) {
			error = EINVAL;
			goto done;
		}
#endif /* SKYWALK */
		fakeif->iff_peer = peer;
		peer_fakeif->iff_peer = ifp;
		connected = TRUE;
	} else if (fakeif->iff_peer != NULL) {
		/* disconnect from peer */
		if_fake_ref   peer_fakeif;

		peer = fakeif->iff_peer;
		peer_fakeif = ifnet_get_if_fake(peer);
		if (peer_fakeif == NULL) {
			/* should not happen */
			error = EINVAL;
			goto done;
		}
		fakeif->iff_peer = NULL;
		peer_fakeif->iff_peer = NULL;
		disconnected = TRUE;
	}

done:
	feth_unlock();

	/* generate link status event if we connect or disconnect */
	if (connected) {
		interface_link_event(ifp, KEV_DL_LINK_ON);
		interface_link_event(peer, KEV_DL_LINK_ON);
	} else if (disconnected) {
		interface_link_event(ifp, KEV_DL_LINK_OFF);
		interface_link_event(peer, KEV_DL_LINK_OFF);
	}
	return error;
}

static int
feth_set_media(ifnet_t ifp, struct if_fake_request * iffr)
{
	if_fake_ref     fakeif;
	int             error;

	if (iffr->iffr_media.iffm_count > IF_FAKE_MEDIA_LIST_MAX) {
		/* list is too long */
		return EINVAL;
	}
	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif == NULL) {
		error = EINVAL;
		goto done;
	}
	fakeif->iff_media_count = iffr->iffr_media.iffm_count;
	bcopy(iffr->iffr_media.iffm_list, fakeif->iff_media_list,
	    iffr->iffr_media.iffm_count * sizeof(fakeif->iff_media_list[0]));
#if 0
	/* XXX: "auto-negotiate" active with peer? */
	/* generate link status event? */
	fakeif->iff_media_current = iffr->iffr_media.iffm_current;
#endif
	error = 0;
done:
	feth_unlock();
	return error;
}

static int
if_fake_request_copyin(user_addr_t user_addr,
    struct if_fake_request *iffr, u_int32_t len)
{
	int     error;

	if (user_addr == USER_ADDR_NULL || len < sizeof(*iffr)) {
		error = EINVAL;
		goto done;
	}
	error = copyin(user_addr, iffr, sizeof(*iffr));
	if (error != 0) {
		goto done;
	}
	if (iffr->iffr_reserved[0] != 0 || iffr->iffr_reserved[1] != 0 ||
	    iffr->iffr_reserved[2] != 0 || iffr->iffr_reserved[3] != 0) {
		error = EINVAL;
		goto done;
	}
done:
	return error;
}

static int
feth_set_drvspec(ifnet_t ifp, uint32_t cmd, u_int32_t len,
    user_addr_t user_addr)
{
	int                     error;
	struct if_fake_request  iffr;
	ifnet_t                 peer;

	switch (cmd) {
	case IF_FAKE_S_CMD_SET_PEER:
		error = if_fake_request_copyin(user_addr, &iffr, len);
		if (error != 0) {
			break;
		}
		if (iffr.iffr_peer_name[0] == '\0') {
			error = feth_config(ifp, NULL);
			break;
		}

		/* ensure nul termination */
		iffr.iffr_peer_name[IFNAMSIZ - 1] = '\0';
		peer = ifunit(__unsafe_null_terminated_from_indexable(iffr.iffr_peer_name));
		if (peer == NULL) {
			error = ENXIO;
			break;
		}
		if (ifnet_type(peer) != IFT_ETHER) {
			error = EINVAL;
			break;
		}
		if (strcmp(ifnet_name(peer), FAKE_ETHER_NAME) != 0) {
			error = EINVAL;
			break;
		}
		error = feth_config(ifp, peer);
		break;
	case IF_FAKE_S_CMD_SET_MEDIA:
		error = if_fake_request_copyin(user_addr, &iffr, len);
		if (error != 0) {
			break;
		}
		error = feth_set_media(ifp, &iffr);
		break;
	case IF_FAKE_S_CMD_SET_DEQUEUE_STALL:
		error = if_fake_request_copyin(user_addr, &iffr, len);
		if (error != 0) {
			break;
		}
		error = feth_enable_dequeue_stall(ifp,
		    iffr.iffr_dequeue_stall);
		break;
	default:
		error = EOPNOTSUPP;
		break;
	}
	return error;
}

static int
feth_get_drvspec(ifnet_t ifp, u_int32_t cmd, u_int32_t len,
    user_addr_t user_addr)
{
	int                     error = EOPNOTSUPP;
	if_fake_ref             fakeif;
	struct if_fake_request  iffr;
	ifnet_t                 peer;

	switch (cmd) {
	case IF_FAKE_G_CMD_GET_PEER:
		if (len < sizeof(iffr)) {
			error = EINVAL;
			break;
		}
		feth_lock();
		fakeif = ifnet_get_if_fake(ifp);
		if (fakeif == NULL) {
			feth_unlock();
			error = EOPNOTSUPP;
			break;
		}
		peer = fakeif->iff_peer;
		feth_unlock();
		bzero(&iffr, sizeof(iffr));
		if (peer != NULL) {
			strlcpy(iffr.iffr_peer_name,
			    if_name(peer),
			    sizeof(iffr.iffr_peer_name));
		}
		error = copyout(&iffr, user_addr, sizeof(iffr));
		break;
	default:
		break;
	}
	return error;
}

union ifdrvu {
	struct ifdrv32  *ifdrvu_32;
	struct ifdrv64  *ifdrvu_64;
	void            *ifdrvu_p;
};

static int
feth_ioctl(ifnet_t ifp, u_long cmd, void * data)
{
	unsigned int            count;
	struct ifdevmtu *       devmtu_p;
	union ifdrvu            drv;
	uint32_t                drv_cmd;
	uint32_t                drv_len;
	boolean_t               drv_set_command = FALSE;
	int                     error = 0;
	struct ifmediareq32 *   ifmr;
	struct ifreq *          ifr;
	if_fake_ref             fakeif;
	int                     status;
	user_addr_t             user_addr;

	ifr = (struct ifreq *)data;
	switch (cmd) {
	case SIOCSIFADDR:
		ifnet_set_flags(ifp, IFF_UP, IFF_UP);
		break;

	case SIOCGIFMEDIA32:
	case SIOCGIFMEDIA64:
		feth_lock();
		fakeif = ifnet_get_if_fake(ifp);
		if (fakeif == NULL) {
			feth_unlock();
			return EOPNOTSUPP;
		}
		status = (fakeif->iff_peer != NULL)
		    ? (IFM_AVALID | IFM_ACTIVE) : IFM_AVALID;
		ifmr = (struct ifmediareq32 *)data;
		user_addr = (cmd == SIOCGIFMEDIA64) ?
		    ((struct ifmediareq64 *)data)->ifmu_ulist :
		    CAST_USER_ADDR_T(((struct ifmediareq32 *)data)->ifmu_ulist);
		count = ifmr->ifm_count;
		ifmr->ifm_active = (fakeif->iff_peer != NULL)
		    ? FAKE_DEFAULT_MEDIA : IFM_ETHER;
		ifmr->ifm_current = IFM_ETHER;
		ifmr->ifm_mask = 0;
		ifmr->ifm_status = status;
		if (user_addr == USER_ADDR_NULL) {
			ifmr->ifm_count = fakeif->iff_media_count;
		} else if (count > 0) {
			if (count > fakeif->iff_media_count) {
				count = fakeif->iff_media_count;
			}
			ifmr->ifm_count = count;
			error = copyout(&fakeif->iff_media_list, user_addr,
			    count * sizeof(int));
		}
		feth_unlock();
		break;

	case SIOCGIFDEVMTU:
		devmtu_p = &ifr->ifr_devmtu;
		devmtu_p->ifdm_current = ifnet_mtu(ifp);
		devmtu_p->ifdm_max = feth_max_mtu(ifp);
		devmtu_p->ifdm_min = IF_MINMTU;
		break;

	case SIOCSIFMTU:
		if ((unsigned int)ifr->ifr_mtu > feth_max_mtu(ifp) ||
		    ifr->ifr_mtu < IF_MINMTU) {
			error = EINVAL;
		} else {
			error = ifnet_set_mtu(ifp, ifr->ifr_mtu);
		}
		break;

	case SIOCSDRVSPEC32:
	case SIOCSDRVSPEC64:
		error = proc_suser(current_proc());
		if (error != 0) {
			break;
		}
		drv_set_command = TRUE;
		OS_FALLTHROUGH;
	case SIOCGDRVSPEC32:
	case SIOCGDRVSPEC64:
		drv.ifdrvu_p = data;
		if (cmd == SIOCGDRVSPEC32 || cmd == SIOCSDRVSPEC32) {
			drv_cmd = drv.ifdrvu_32->ifd_cmd;
			drv_len = drv.ifdrvu_32->ifd_len;
			user_addr = CAST_USER_ADDR_T(drv.ifdrvu_32->ifd_data);
		} else {
			drv_cmd = drv.ifdrvu_64->ifd_cmd;
			drv_len = drv.ifdrvu_64->ifd_len;
			user_addr = drv.ifdrvu_64->ifd_data;
		}
		if (drv_set_command) {
			error = feth_set_drvspec(ifp, drv_cmd, drv_len,
			    user_addr);
		} else {
			error = feth_get_drvspec(ifp, drv_cmd, drv_len,
			    user_addr);
		}
		break;

	case SIOCSIFLLADDR:
		error = ifnet_set_lladdr(ifp, ifr->ifr_addr.sa_data,
		    ifr->ifr_addr.sa_len);
		break;

	case SIOCSIFFLAGS:
		if ((ifp->if_flags & IFF_UP) != 0) {
			/* marked up, set running if not already set */
			if ((ifp->if_flags & IFF_RUNNING) == 0) {
				/* set running */
				error = ifnet_set_flags(ifp, IFF_RUNNING,
				    IFF_RUNNING);
			}
		} else if ((ifp->if_flags & IFF_RUNNING) != 0) {
			/* marked down, clear running */
			error = ifnet_set_flags(ifp, 0, IFF_RUNNING);
		}
		break;

	case SIOCADDMULTI:
	case SIOCDELMULTI:
		error = 0;
		break;
	case SIOCSIFCAP: {
		uint32_t        cap;

		feth_lock();
		fakeif = ifnet_get_if_fake(ifp);
		if (fakeif == NULL ||
		    (fakeif->iff_flags & IFF_FLAGS_LRO) == 0) {
			feth_unlock();
			return EOPNOTSUPP;
		}
		feth_unlock();
		cap = (ifr->ifr_reqcap & IFCAP_LRO) != 0 ? IFCAP_LRO : 0;
		error = ifnet_set_capabilities_enabled(ifp, cap, IFCAP_LRO);
		break;
	}
	default:
		error = EOPNOTSUPP;
		break;
	}
	return error;
}

static void
feth_if_free(ifnet_t ifp)
{
	if_fake_ref           fakeif;

	if (ifp == NULL) {
		return;
	}
	feth_lock();
	fakeif = ifnet_get_if_fake(ifp);
	if (fakeif == NULL) {
		feth_unlock();
		return;
	}
	ifp->if_softc = NULL;
#if SKYWALK
	VERIFY(fakeif->iff_doorbell_tcall == NULL);
#endif /* SKYWALK */
	feth_unlock();
	feth_release(fakeif);
	ifnet_release(ifp);
	return;
}

__private_extern__ void
if_fake_init(void)
{
	int error;

#if SKYWALK
	(void)feth_register_nexus_domain_provider();
#endif /* SKYWALK */
	error = if_clone_attach(&feth_cloner);
	if (error != 0) {
		return;
	}
	return;
}