d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

bnxt: Choose better HW LRO defaults for performance

Browse Source 
1) Choose correct Firmware options for HW LRO for best performance
2) Delete TBD and other comments which are not required.
3) Added sysctl interface to enable / disable / modify different factors
   of HW LRO.
4) Disabled HW LRO by default to avoid issues with packet forwarding

This allows much better control over the LRO configuration via sysctls, and
uses much better defaults.  Hardware LRO can now be enabled/disabled
independantly from the software LRO, and the tuning parameters are exposed.

manpage updates coming soon.

Submitted by:	Bhargava Chenna Marreddy <bhargava.marreddy@broadcom.com>
Reviewed by:	shurd, sbruno
Approved by:	sbruno (mentor)
Sponsored by:	Broadcom Limited
Differential Revision:	https://reviews.freebsd.org/D12223
This commit is contained in:
Stephen Hurd 2017-09-23 16:59:37 +00:00
parent 1225d9da9f
commit 59059cda6a
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=323955
6 changed files with 156 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
sys/dev/bnxt/bnxt.h
View File

@ -526,6 +526,14 @@ struct bnxt_func_qcfg {
uint16_t alloc_vnics;
};
struct bnxt_hw_lro {
uint16_t enable;
uint16_t is_mode_gro;
uint16_t max_agg_segs;
uint16_t max_aggs;
uint32_t min_agg_len;
};
struct bnxt_softc {
device_t dev;
if_ctx_t ctx;
@ -586,10 +594,13 @@ struct bnxt_softc {
struct sysctl_ctx_list hw_stats;
struct sysctl_oid *hw_stats_oid;
struct sysctl_ctx_list hw_lro_ctx;
struct sysctl_oid *hw_lro_oid;
struct bnxt_ver_info *ver_info;
struct bnxt_nvram_info *nvm_info;
bool wol;
struct bnxt_hw_lro hw_lro;
uint8_t wol_filter_id;
uint16_t rx_coal_usecs;
uint16_t rx_coal_usecs_irq;

58
sys/dev/bnxt/bnxt_hwrm.c
View File

@ -977,29 +977,53 @@ bnxt_cfg_async_cr(struct bnxt_softc *softc)
return rc;
}
void
bnxt_validate_hw_lro_settings(struct bnxt_softc *softc)
{
softc->hw_lro.enable = min(softc->hw_lro.enable, 1);
softc->hw_lro.is_mode_gro = min(softc->hw_lro.is_mode_gro, 1);
softc->hw_lro.max_agg_segs = min(softc->hw_lro.max_agg_segs,
HWRM_VNIC_TPA_CFG_INPUT_MAX_AGG_SEGS_MAX);
softc->hw_lro.max_aggs = min(softc->hw_lro.max_aggs,
HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX);
softc->hw_lro.min_agg_len = min(softc->hw_lro.min_agg_len, BNXT_MAX_MTU);
}
int
bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic,
uint32_t flags)
bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc)
{
struct hwrm_vnic_tpa_cfg_input req = {0};
uint32_t flags;
bnxt_hwrm_cmd_hdr_init(softc, &req, HWRM_VNIC_TPA_CFG);
req.flags = htole32(flags);
req.vnic_id = htole16(vnic->id);
req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS |
HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS |
/* HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_TIMER | */
HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
/* TODO: Calculate this based on ring size? */
req.max_agg_segs = htole16(3);
/* Base this in the allocated TPA start size... */
req.max_aggs = htole16(7);
/*
* TODO: max_agg_timer?
* req.mag_agg_timer = htole32(XXX);
*/
req.min_agg_len = htole32(0);
if (softc->hw_lro.enable) {
flags = HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA |
HWRM_VNIC_TPA_CFG_INPUT_FLAGS_ENCAP_TPA |
HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_ECN |
HWRM_VNIC_TPA_CFG_INPUT_FLAGS_AGG_WITH_SAME_GRE_SEQ;
if (softc->hw_lro.is_mode_gro)
flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_GRO;
else
flags |= HWRM_VNIC_TPA_CFG_INPUT_FLAGS_RSC_WND_UPDATE;
req.flags = htole32(flags);
req.enables = htole32(HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGG_SEGS |
HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MAX_AGGS |
HWRM_VNIC_TPA_CFG_INPUT_ENABLES_MIN_AGG_LEN);
req.max_agg_segs = htole16(softc->hw_lro.max_agg_segs);
req.max_aggs = htole16(softc->hw_lro.max_aggs);
req.min_agg_len = htole32(softc->hw_lro.min_agg_len);
}
req.vnic_id = htole16(softc->vnic_info.id);
return hwrm_send_message(softc, &req, sizeof(req));
}

4
sys/dev/bnxt/bnxt_hwrm.h
View File

@ -62,8 +62,8 @@ int bnxt_hwrm_set_filter(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic);
int bnxt_hwrm_rss_cfg(struct bnxt_softc *softc, struct bnxt_vnic_info *vnic,
uint32_t hash_type);
int bnxt_cfg_async_cr(struct bnxt_softc *softc);
int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc,
struct bnxt_vnic_info *vnic, uint32_t flags);
int bnxt_hwrm_vnic_tpa_cfg(struct bnxt_softc *softc);
void bnxt_validate_hw_lro_settings(struct bnxt_softc *softc);
int bnxt_hwrm_nvm_find_dir_entry(struct bnxt_softc *softc, uint16_t type,
uint16_t *ordinal, uint16_t ext, uint16_t *index, bool use_index,
uint8_t search_opt, uint32_t *data_length, uint32_t *item_length,

85
sys/dev/bnxt/bnxt_sysctl.c
View File

@ -84,6 +84,16 @@ bnxt_init_sysctl_ctx(struct bnxt_softc *softc)
return ENOMEM;
}
sysctl_ctx_init(&softc->hw_lro_ctx);
ctx = device_get_sysctl_ctx(softc->dev);
softc->hw_lro_oid = SYSCTL_ADD_NODE(ctx,
SYSCTL_CHILDREN(device_get_sysctl_tree(softc->dev)), OID_AUTO,
"hw_lro", CTLFLAG_RD, 0, "hardware lro");
if (!softc->hw_lro_oid) {
sysctl_ctx_free(&softc->hw_lro_ctx);
return ENOMEM;
}
return 0;
}
@ -114,6 +124,13 @@ bnxt_free_sysctl_ctx(struct bnxt_softc *softc)
else
softc->nvm_info->nvm_oid = NULL;
}
if (softc->hw_lro_oid != NULL) {
orc = sysctl_ctx_free(&softc->hw_lro_ctx);
if (orc)
rc = orc;
else
softc->hw_lro_oid = NULL;
}
return rc;
}
@ -1210,6 +1227,74 @@ bnxt_create_config_sysctls_pre(struct bnxt_softc *softc)
return 0;
}
#define BNXT_HW_LRO_FN(fn_name, arg) \
static int \
fn_name(SYSCTL_HANDLER_ARGS) { \
struct bnxt_softc *softc = arg1; \
int rc; \
int val; \
\
if (softc == NULL) \
return EBUSY; \
\
val = softc->hw_lro.arg; \
rc = sysctl_handle_int(oidp, &val, 0, req); \
if (rc || !req->newptr) \
return rc; \
\
if ((if_getdrvflags(iflib_get_ifp(softc->ctx)) & IFF_DRV_RUNNING)) \
return EBUSY; \
\
softc->hw_lro.arg = val; \
bnxt_validate_hw_lro_settings(softc); \
rc = bnxt_hwrm_vnic_tpa_cfg(softc); \
\
return rc; \
}
BNXT_HW_LRO_FN(bnxt_hw_lro_enable_disable, enable)
BNXT_HW_LRO_FN(bnxt_hw_lro_set_mode, is_mode_gro)
BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_agg_segs, max_agg_segs)
BNXT_HW_LRO_FN(bnxt_hw_lro_set_max_aggs, max_aggs)
BNXT_HW_LRO_FN(bnxt_hw_lro_set_min_agg_len, min_agg_len)
int
bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc)
{
struct sysctl_oid *oid = softc->hw_lro_oid;
if (!oid)
return ENOMEM;
SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
"enable", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
bnxt_hw_lro_enable_disable, "A",
"Enable or Disable HW LRO: 0 / 1");
SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
"gro_mode", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
bnxt_hw_lro_set_mode, "A",
"Set mode: 1 = GRO mode, 0 = RSC mode");
SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
"max_agg_segs", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
bnxt_hw_lro_set_max_agg_segs, "A",
"Set Max Agg Seg Value (unit is Log2): "
"0 (= 1 seg) / 1 (= 2 segs) / ... / 31 (= 2^31 segs)");
SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
"max_aggs", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
bnxt_hw_lro_set_max_aggs, "A",
"Set Max Aggs Value (unit is Log2): "
"0 (= 1 agg) / 1 (= 2 aggs) / ... / 7 (= 2^7 segs)");
SYSCTL_ADD_PROC(&softc->hw_lro_ctx, SYSCTL_CHILDREN(oid), OID_AUTO,
"min_agg_len", CTLTYPE_INT|CTLFLAG_RWTUN, softc, 0,
bnxt_hw_lro_set_min_agg_len, "A",
"Min Agg Len: 1 to 9000");
return 0;
}
static int
bnxt_vlan_only_sysctl(SYSCTL_HANDLER_ARGS) {
struct bnxt_softc *softc = arg1;

1
sys/dev/bnxt/bnxt_sysctl.h
View File

@ -40,3 +40,4 @@ int bnxt_create_ver_sysctls(struct bnxt_softc *softc);
int bnxt_create_nvram_sysctls(struct bnxt_nvram_info *ni);
int bnxt_create_config_sysctls_pre(struct bnxt_softc *softc);
int bnxt_create_config_sysctls_post(struct bnxt_softc *softc);
int bnxt_create_hw_lro_sysctls(struct bnxt_softc *softc);

25
sys/dev/bnxt/if_bnxt.c
View File

@ -826,6 +826,17 @@ bnxt_attach_pre(if_ctx_t ctx)
/* iflib will map and release this bar */
scctx->isc_msix_bar = pci_msix_table_bar(softc->dev);
/*
* Default settings for HW LRO (TPA):
* Disable HW LRO by default
* Can be enabled after taking care of 'packet forwarding'
*/
softc->hw_lro.enable = 0;
softc->hw_lro.is_mode_gro = 0;
softc->hw_lro.max_agg_segs = 5; /* 2^5 = 32 segs */
softc->hw_lro.max_aggs = HWRM_VNIC_TPA_CFG_INPUT_MAX_AGGS_MAX;
softc->hw_lro.min_agg_len = 512;
/* Allocate the default completion ring */
softc->def_cp_ring.stats_ctx_id = HWRM_NA_SIGNATURE;
softc->def_cp_ring.ring.phys_id = (uint16_t)HWRM_NA_SIGNATURE;
@ -861,6 +872,10 @@ bnxt_attach_pre(if_ctx_t ctx)
if (rc)
goto failed;
rc = bnxt_create_hw_lro_sysctls(softc);
if (rc)
goto failed;
/* Initialize the vlan list */
SLIST_INIT(&softc->vnic_info.vlan_tags);
softc->vnic_info.vlan_tag_list.idi_vaddr = NULL;
@ -1071,15 +1086,7 @@ bnxt_init(if_ctx_t ctx)
if (rc)
goto fail;
/*
* Enable LRO/TPA/GRO
* TBD:
* Enable / Disable HW_LRO based on
* ifconfig lro / ifconfig -lro setting
*/
rc = bnxt_hwrm_vnic_tpa_cfg(softc, &softc->vnic_info,
(if_getcapenable(iflib_get_ifp(ctx)) & IFCAP_LRO) ?
HWRM_VNIC_TPA_CFG_INPUT_FLAGS_TPA : 0);
rc = bnxt_hwrm_vnic_tpa_cfg(softc);
if (rc)
goto fail;