sfxge(4): improve PCIe link speed and width check

Perform a more accurate check of whether the PCIe bandwidth is
sufficient for the current/supported port modes.

Give a different warning if there is sufficient bandwidth to achieve
line rate, but the link is not fast enough for optimal latency.

Submitted by:   Mark Spender <mspender at solarflare.com>
Sponsored by:   Solarflare Communications, Inc.
MFC after:      1 week
Differential Revision:  https://reviews.freebsd.org/D6369

sys/dev/sfxge/common/ef10_impl.h

@@ -1035,7 +1035,13 @@ efx_mcdi_get_port_assignment(
 extern	__checkReturn	efx_rc_t
 efx_mcdi_get_port_modes(
 	__in		efx_nic_t *enp,
-	__out		uint32_t *modesp);
+	__out		uint32_t *modesp,
+	__out_opt	uint32_t *current_modep);
+
+extern	__checkReturn	efx_rc_t
+ef10_nic_get_port_mode_bandwidth(
+	__in		uint32_t port_mode,
+	__out		uint32_t *bandwidth_mbpsp);
 
 extern	__checkReturn	efx_rc_t
 efx_mcdi_get_mac_address_pf(

sys/dev/sfxge/common/ef10_nic.c

@@ -88,7 +88,8 @@ efx_mcdi_get_port_assignment(
 	__checkReturn	efx_rc_t
 efx_mcdi_get_port_modes(
 	__in		efx_nic_t *enp,
-	__out		uint32_t *modesp)
+	__out		uint32_t *modesp,
+	__out_opt	uint32_t *current_modep)
 {
 	efx_mcdi_req_t req;
 	uint8_t payload[MAX(MC_CMD_GET_PORT_MODES_IN_LEN,
@@ -113,19 +114,31 @@ efx_mcdi_get_port_modes(
 	}
 
 	/*
-	 * Require only Modes and DefaultMode fields.
+	 * Require only Modes and DefaultMode fields, unless the current mode
-	 * (CurrentMode field was added for Medford)
+	 * was requested (CurrentMode field was added for Medford).
 	 */
 	if (req.emr_out_length_used <
 	    MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST) {
 		rc = EMSGSIZE;
 		goto fail2;
 	}
+	if ((current_modep != NULL) && (req.emr_out_length_used <
+	    MC_CMD_GET_PORT_MODES_OUT_CURRENT_MODE_OFST + 4)) {
+		rc = EMSGSIZE;
+		goto fail3;
+	}
 
 	*modesp = MCDI_OUT_DWORD(req, GET_PORT_MODES_OUT_MODES);
 
+	if (current_modep != NULL) {
+		*current_modep = MCDI_OUT_DWORD(req,
+					    GET_PORT_MODES_OUT_CURRENT_MODE);
+	}
+
 	return (0);
 
+fail3:
+	EFSYS_PROBE(fail3);
 fail2:
 	EFSYS_PROBE(fail2);
 fail1:
@@ -134,6 +147,50 @@ efx_mcdi_get_port_modes(
 	return (rc);
 }
 
+	__checkReturn	efx_rc_t
+ef10_nic_get_port_mode_bandwidth(
+	__in		uint32_t port_mode,
+	__out		uint32_t *bandwidth_mbpsp)
+{
+	uint32_t bandwidth;
+	efx_rc_t rc;
+
+	switch (port_mode) {
+	case TLV_PORT_MODE_10G:
+		bandwidth = 10000;
+		break;
+	case TLV_PORT_MODE_10G_10G:
+		bandwidth = 10000 * 2;
+		break;
+	case TLV_PORT_MODE_10G_10G_10G_10G:
+	case TLV_PORT_MODE_10G_10G_10G_10G_Q:
+	case TLV_PORT_MODE_10G_10G_10G_10G_Q2:
+		bandwidth = 10000 * 4;
+		break;
+	case TLV_PORT_MODE_40G:
+		bandwidth = 40000;
+		break;
+	case TLV_PORT_MODE_40G_40G:
+		bandwidth = 40000 * 2;
+		break;
+	case TLV_PORT_MODE_40G_10G_10G:
+	case TLV_PORT_MODE_10G_10G_40G:
+		bandwidth = 40000 + (10000 * 2);
+		break;
+	default:
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	*bandwidth_mbpsp = bandwidth;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
 
 static	__checkReturn		efx_rc_t
 efx_mcdi_vadaptor_alloc(
@@ -1090,7 +1147,7 @@ ef10_external_port_mapping(
 	uint32_t matches;
 	uint32_t stride = 1; /* default 1-1 mapping */
 
-	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes)) != 0) {
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, NULL)) != 0) {
 		/* No port mode information available - use default mapping */
 		goto out;
 	}

sys/dev/sfxge/common/efx.h

@@ -173,6 +173,30 @@ extern 		void
 efx_nic_destroy(
 	__in	efx_nic_t *enp);
 
+#define	EFX_PCIE_LINK_SPEED_GEN1		1
+#define	EFX_PCIE_LINK_SPEED_GEN2		2
+#define	EFX_PCIE_LINK_SPEED_GEN3		3
+
+typedef enum efx_pcie_link_performance_e {
+	EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH,
+	EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH,
+	EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY,
+	EFX_PCIE_LINK_PERFORMANCE_OPTIMAL
+} efx_pcie_link_performance_t;
+
+extern	__checkReturn	efx_rc_t
+efx_nic_calculate_pcie_link_bandwidth(
+	__in		uint32_t pcie_link_width,
+	__in		uint32_t pcie_link_gen,
+	__out		uint32_t *bandwidth_mbpsp);
+
+extern	__checkReturn	efx_rc_t
+efx_nic_check_pcie_link_speed(
+	__in		efx_nic_t *enp,
+	__in		uint32_t pcie_link_width,
+	__in		uint32_t pcie_link_gen,
+	__out		efx_pcie_link_performance_t *resultp);
+
 #if EFSYS_OPT_MCDI
 
 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD
@@ -1116,6 +1140,9 @@ typedef struct efx_nic_cfg_s {
 	uint32_t		enc_mcdi_max_payload_length;
 	/* VPD may be per-PF or global */
 	boolean_t		enc_vpd_is_global;
+	/* Minimum unidirectional bandwidth in Mb/s to max out all ports */
+	uint32_t		enc_required_pcie_bandwidth_mbps;
+	uint32_t		enc_max_pcie_link_gen;
 } efx_nic_cfg_t;
 
 #define	EFX_PCI_FUNCTION_IS_PF(_encp)	((_encp)->enc_vf == 0xffff)

sys/dev/sfxge/common/efx_nic.c

@@ -964,3 +964,101 @@ efx_mcdi_get_loopback_modes(
 }
 
 #endif /* EFSYS_OPT_LOOPBACK */
+
+	__checkReturn	efx_rc_t
+efx_nic_calculate_pcie_link_bandwidth(
+	__in		uint32_t pcie_link_width,
+	__in		uint32_t pcie_link_gen,
+	__out		uint32_t *bandwidth_mbpsp)
+{
+	uint32_t lane_bandwidth;
+	uint32_t total_bandwidth;
+	efx_rc_t rc;
+
+	if ((pcie_link_width == 0) || (pcie_link_width > 16) ||
+	    !ISP2(pcie_link_width)) {
+		rc = EINVAL;
+		goto fail1;
+	}
+
+	switch (pcie_link_gen) {
+	case EFX_PCIE_LINK_SPEED_GEN1:
+		/* 2.5 Gb/s raw bandwidth with 8b/10b encoding */
+		lane_bandwidth = 2000;
+		break;
+	case EFX_PCIE_LINK_SPEED_GEN2:
+		/* 5.0 Gb/s raw bandwidth with 8b/10b encoding */
+		lane_bandwidth = 4000;
+		break;
+	case EFX_PCIE_LINK_SPEED_GEN3:
+		/* 8.0 Gb/s raw bandwidth with 128b/130b encoding */
+		lane_bandwidth = 7877;
+		break;
+	default:
+		rc = EINVAL;
+		goto fail2;
+	}
+
+	total_bandwidth = lane_bandwidth * pcie_link_width;
+	*bandwidth_mbpsp = total_bandwidth;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
+
+	__checkReturn	efx_rc_t
+efx_nic_check_pcie_link_speed(
+	__in		efx_nic_t *enp,
+	__in		uint32_t pcie_link_width,
+	__in		uint32_t pcie_link_gen,
+	__out		efx_pcie_link_performance_t *resultp)
+{
+	efx_nic_cfg_t *encp = &(enp->en_nic_cfg);
+	uint32_t bandwidth;
+	efx_pcie_link_performance_t result;
+	efx_rc_t rc;
+
+	if ((encp->enc_required_pcie_bandwidth_mbps == 0) ||
+	    (pcie_link_width == 0) || (pcie_link_width == 32) ||
+	    (pcie_link_gen == 0)) {
+		/*
+		 * No usable info on what is required and/or in use. In virtual
+		 * machines, sometimes the PCIe link width is reported as 0 or
+		 * 32, or the speed as 0.
+		 */
+		result = EFX_PCIE_LINK_PERFORMANCE_UNKNOWN_BANDWIDTH;
+		goto out;
+	}
+
+	/* Calculate the available bandwidth in megabits per second */
+	rc = efx_nic_calculate_pcie_link_bandwidth(pcie_link_width,
+					    pcie_link_gen, &bandwidth);
+	if (rc != 0)
+		goto fail1;
+
+	if (bandwidth < encp->enc_required_pcie_bandwidth_mbps) {
+		result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_BANDWIDTH;
+	} else if (pcie_link_gen < encp->enc_max_pcie_link_gen) {
+		/* The link provides enough bandwidth but not optimal latency */
+		result = EFX_PCIE_LINK_PERFORMANCE_SUBOPTIMAL_LATENCY;
+	} else {
+		result = EFX_PCIE_LINK_PERFORMANCE_OPTIMAL;
+	}
+
+out:
+	*resultp = result;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}

sys/dev/sfxge/common/hunt_nic.c

@@ -39,6 +39,65 @@ __FBSDID("$FreeBSD$");
 
 #if EFSYS_OPT_HUNTINGTON
 
+#include "ef10_tlv_layout.h"
+
+static	__checkReturn	efx_rc_t
+hunt_nic_get_required_pcie_bandwidth(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *bandwidth_mbpsp)
+{
+	uint32_t port_modes;
+	uint32_t max_port_mode;
+	uint32_t bandwidth;
+	efx_rc_t rc;
+
+	/*
+	 * On Huntington, the firmware may not give us the current port mode, so
+	 * we need to go by the set of available port modes and assume the most
+	 * capable mode is in use.
+	 */
+
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes, NULL)) != 0) {
+		/* No port mode info available */
+		bandwidth = 0;
+		goto out;
+	}
+
+	if (port_modes & (1 << TLV_PORT_MODE_40G_40G)) {
+		/*
+		 * This needs the full PCIe bandwidth (and could use
+		 * more) - roughly 64 Gbit/s for 8 lanes of Gen3.
+		 */
+		if ((rc = efx_nic_calculate_pcie_link_bandwidth(8,
+			    EFX_PCIE_LINK_SPEED_GEN3, &bandwidth)) != 0)
+			goto fail1;
+	} else {
+		if (port_modes & (1 << TLV_PORT_MODE_40G)) {
+			max_port_mode = TLV_PORT_MODE_40G;
+		} else if (port_modes & (1 << TLV_PORT_MODE_10G_10G_10G_10G)) {
+			max_port_mode = TLV_PORT_MODE_10G_10G_10G_10G;
+		} else {
+			/* Assume two 10G ports */
+			max_port_mode = TLV_PORT_MODE_10G_10G;
+		}
+
+		if ((rc = ef10_nic_get_port_mode_bandwidth(max_port_mode,
+							    &bandwidth)) != 0)
+		    goto fail2;
+	}
+
+out:
+	*bandwidth_mbpsp = bandwidth;
+
+	return (0);
+
+fail2:
+	EFSYS_PROBE(fail2);
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
 
 	__checkReturn	efx_rc_t
 hunt_board_cfg(
@@ -57,6 +116,7 @@ hunt_board_cfg(
 	uint32_t flags;
 	uint32_t sysclk;
 	uint32_t base, nvec;
+	uint32_t bandwidth;
 	efx_rc_t rc;
 
 	if ((rc = efx_mcdi_get_port_assignment(enp, &port)) != 0)
@@ -286,8 +346,17 @@ hunt_board_cfg(
 	 */
 	encp->enc_tx_tso_tcp_header_offset_limit = EF10_TCP_HEADER_OFFSET_LIMIT;
 
+	if ((rc = hunt_nic_get_required_pcie_bandwidth(enp, &bandwidth)) != 0)
+		goto fail15;
+	encp->enc_required_pcie_bandwidth_mbps = bandwidth;
+
+	/* All Huntington devices have a PCIe Gen3, 8 lane connector */
+	encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN3;
+
 	return (0);
 
+fail15:
+	EFSYS_PROBE(fail15);
 fail14:
 	EFSYS_PROBE(fail14);
 fail13:

sys/dev/sfxge/common/medford_nic.c

@@ -95,6 +95,38 @@ efx_mcdi_get_rxdp_config(
 	return (rc);
 }
 
+static	__checkReturn	efx_rc_t
+medford_nic_get_required_pcie_bandwidth(
+	__in		efx_nic_t *enp,
+	__out		uint32_t *bandwidth_mbpsp)
+{
+	uint32_t port_modes;
+	uint32_t current_mode;
+	uint32_t bandwidth;
+	efx_rc_t rc;
+
+	if ((rc = efx_mcdi_get_port_modes(enp, &port_modes,
+				    &current_mode)) != 0) {
+		/* No port mode info available. */
+		bandwidth = 0;
+		goto out;
+	}
+
+	if ((rc = ef10_nic_get_port_mode_bandwidth(current_mode,
+						    &bandwidth)) != 0)
+		goto fail1;
+
+out:
+	*bandwidth_mbpsp = bandwidth;
+
+	return (0);
+
+fail1:
+	EFSYS_PROBE1(fail1, efx_rc_t, rc);
+
+	return (rc);
+}
+
 	__checkReturn	efx_rc_t
 medford_board_cfg(
 	__in		efx_nic_t *enp)
@@ -112,6 +144,7 @@ medford_board_cfg(
 	uint32_t sysclk;
 	uint32_t base, nvec;
 	uint32_t end_padding;
+	uint32_t bandwidth;
 	efx_rc_t rc;
 
 	/*
@@ -275,8 +308,16 @@ medford_board_cfg(
 	 */
 	encp->enc_vpd_is_global = B_TRUE;
 
+	rc = medford_nic_get_required_pcie_bandwidth(enp, &bandwidth);
+	if (rc != 0)
+		goto fail13;
+	encp->enc_required_pcie_bandwidth_mbps = bandwidth;
+	encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN3;
+
 	return (0);
 
+fail13:
+	EFSYS_PROBE(fail13);
 fail12:
 	EFSYS_PROBE(fail12);
 fail11:

sys/dev/sfxge/common/siena_nic.c

@@ -150,6 +150,10 @@ siena_board_cfg(
 	encp->enc_fw_assisted_tso_v2_enabled = B_FALSE;
 	encp->enc_allow_set_mac_with_installed_filters = B_TRUE;
 
+	/* Siena supports two 10G ports, and 8 lanes of PCIe Gen2 */
+	encp->enc_required_pcie_bandwidth_mbps = 2 * 10000;
+	encp->enc_max_pcie_link_gen = EFX_PCIE_LINK_SPEED_GEN2;
+
 	return (0);
 
 fail2: