d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make igb and ixgbe check tunables at probe time.

Browse Source 
This allows one to make a kernel module to tune the
number of queues before the driver loads.

This is needed so that a module at SI_SUB_CPU can set
tunables for these drivers to take.  Otherwise getenv
is called too early by the TUNABLE macros.

Reviewed by: smh
Phabric: https://reviews.freebsd.org/D1149
This commit is contained in:
Alfred Perlstein 2014-11-26 20:19:36 +00:00
parent b05765d75f
commit 56c14bca7e
4 changed files with 100 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
sys/dev/e1000/if_igb.c
View File

@ -188,6 +188,7 @@ static char *igb_strings[] = {
/*********************************************************************
* Function prototypes
*********************************************************************/
static int igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS);
static int igb_probe(device_t);
static int igb_attach(device_t);
static int igb_detach(device_t);
@ -493,6 +494,11 @@ igb_attach(device_t dev)
OID_AUTO, "nvm", CTLTYPE_INT|CTLFLAG_RW, adapter, 0,
igb_sysctl_nvm_info, "I", "NVM Information");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "num_queues", CTLTYPE_INT | CTLFLAG_RD,
adapter, 0, igb_per_unit_num_queues, "I", "Number of Queues");
igb_set_sysctl_value(adapter, "enable_aim",
"Interrupt Moderation", &adapter->enable_aim,
igb_enable_aim);
@ -2831,6 +2837,7 @@ igb_setup_msix(struct adapter *adapter)
{
device_t dev = adapter->dev;
int bar, want, queues, msgs, maxqueues;
int n_queues;
/* tuneable override */
if (igb_enable_msix == 0)
@ -2858,8 +2865,18 @@ igb_setup_msix(struct adapter *adapter)
goto msi;
}
/* Figure out a reasonable auto config value */
queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
n_queues = 0;
/* try more specific tunable, then global, then finally default to boot time tunable if set. */
if (device_getenv_int(dev, "num_queues", &n_queues) != 0) {
device_printf(dev, "using specific tunable num_queues=%d", n_queues);
} else if (TUNABLE_INT_FETCH("hw.igb.num_queues", &n_queues) != 0) {
if (igb_num_queues != n_queues) {
device_printf(dev, "using global tunable hw.igb.num_queues=%d", n_queues);
igb_num_queues = n_queues;
}
} else {
n_queues = igb_num_queues;
}
#ifdef RSS
/* If we're doing RSS, clamp at the number of RSS buckets */
@ -2867,10 +2884,12 @@ igb_setup_msix(struct adapter *adapter)
queues = rss_getnumbuckets();
#endif
/* Manual override */
if (igb_num_queues != 0)
queues = igb_num_queues;
if (n_queues != 0) {
queues = n_queues;
} else {
/* Figure out a reasonable auto config value */
queues = (mp_ncpus > (msgs-1)) ? (msgs-1) : mp_ncpus;
}
/* Sanity check based on HW */
switch (adapter->hw.mac.type) {
@ -2893,12 +2912,17 @@ igb_setup_msix(struct adapter *adapter)
maxqueues = 1;
break;
}
if (queues > maxqueues)
if (queues > maxqueues) {
device_printf(adapter->dev, "requested %d queues, but max for this adapter is %d\n",
queues, maxqueues);
queues = maxqueues;
/* Manual override */
if (igb_num_queues != 0)
queues = igb_num_queues;
} else if (queues == 0) {
queues = 1;
} else if (queues < 0) {
device_printf(adapter->dev, "requested %d queues, but min for this adapter is %d\n",
queues, 1);
queues = 1;
}
/*
** One vector (RX/TX pair) per queue
@ -6384,3 +6408,14 @@ igb_sysctl_eee(SYSCTL_HANDLER_ARGS)
IGB_CORE_UNLOCK(adapter);
return (0);
}
static int
igb_per_unit_num_queues(SYSCTL_HANDLER_ARGS)
{
struct adapter *adapter;
adapter = (struct adapter *) arg1;
return sysctl_handle_int(oidp, &adapter->num_queues, 0, req);
}

41
sys/dev/ixgbe/ixgbe.c
View File

@ -103,6 +103,7 @@ static char *ixgbe_strings[] = {
/*********************************************************************
* Function prototypes
*********************************************************************/
static int ixgbe_per_unit_num_queues(SYSCTL_HANDLER_ARGS);
static int ixgbe_probe(device_t);
static int ixgbe_attach(device_t);
static int ixgbe_detach(device_t);
@ -473,6 +474,11 @@ ixgbe_attach(device_t dev)
OID_AUTO, "advertise_speed", CTLTYPE_INT | CTLFLAG_RW,
adapter, 0, ixgbe_set_advertise, "I", "Link Speed");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "num_queues", CTLTYPE_INT | CTLFLAG_RD,
adapter, 0, ixgbe_per_unit_num_queues, "I", "Number of Queues");
SYSCTL_ADD_PROC(device_get_sysctl_ctx(dev),
SYSCTL_CHILDREN(device_get_sysctl_tree(dev)),
OID_AUTO, "ts", CTLTYPE_INT | CTLFLAG_RW, adapter,
@ -2516,6 +2522,7 @@ ixgbe_setup_msix(struct adapter *adapter)
{
device_t dev = adapter->dev;
int rid, want, queues, msgs;
int n_queues;
/* Override by tuneable */
if (ixgbe_enable_msix == 0)
@ -2548,15 +2555,29 @@ ixgbe_setup_msix(struct adapter *adapter)
queues = rss_getnumbuckets();
#endif
if (ixgbe_num_queues != 0)
queues = ixgbe_num_queues;
/* try more specific tunable, then global, then finally default to boot time tunable if set. */
if (device_getenv_int(dev, "num_queues", &n_queues) != 0) {
device_printf(dev, "using specific tunable numqueues=%d", n_queues);
} else if (TUNABLE_INT_FETCH("hw.ix.num_queues", &n_queues) != 0) {
if (ixgbe_num_queues != n_queues) {
device_printf(dev, "using global tunable num_queues=%d", n_queues);
ixgbe_num_queues = n_queues;
}
} else {
n_queues = ixgbe_num_queues;
}
if (n_queues < 0) {
device_printf(dev, "tunable < 0, resetting to default");
n_queues = 0;
}
if (n_queues != 0)
queues = n_queues;
/* Set max queues to 8 when autoconfiguring */
else if ((ixgbe_num_queues == 0) && (queues > 8))
queues = 8;
/* reflect correct sysctl value */
ixgbe_num_queues = queues;
/*
** Want one vector (RX/TX pair) per queue
** plus an additional for Link.
@ -5943,6 +5964,16 @@ ixgbe_set_flowcntl(SYSCTL_HANDLER_ARGS)
return error;
}
static int
ixgbe_per_unit_num_queues(SYSCTL_HANDLER_ARGS)
{
struct adapter *adapter;
adapter = (struct adapter *) arg1;
return sysctl_handle_int(oidp, &adapter->num_queues, 0, req);
}
/*
** Control link advertise speed:
** 1 - advertise only 1G

17
sys/kern/subr_bus.c
View File

@ -56,6 +56,8 @@ __FBSDID("$FreeBSD$");
#include <sys/interrupt.h>
#include <sys/cpuset.h>
#include <sys/syslog.h>
#include <net/vnet.h>
#include <machine/cpu.h>
@ -5031,3 +5033,18 @@ bus_free_resource(device_t dev, int type, struct resource *r)
return (0);
return (bus_release_resource(dev, type, rman_get_rid(r), r));
}
int
device_getenv_int(device_t dev, const char *knob, int *iptr)
{
char env[128];
int sz;
sz = snprintf(env, sizeof(env), "hw.%s.%d.%s", device_get_name(dev), device_get_unit(dev), knob);
if (sz >= sizeof(env)) {
/* XXX: log? return error? bump sysctl error? */
log(LOG_ERR, "device_getenv_int: knob too long: '%s'", knob);
return 0;
}
return (getenv_int(env, iptr));
}

1
sys/sys/bus.h
View File

@ -449,6 +449,7 @@ const char *device_get_nameunit(device_t dev);
void *device_get_softc(device_t dev);
device_state_t device_get_state(device_t dev);
int device_get_unit(device_t dev);
int device_getenv_int(device_t dev, const char *knob, int *iptr);
struct sysctl_ctx_list *device_get_sysctl_ctx(device_t dev);
struct sysctl_oid *device_get_sysctl_tree(device_t dev);
int device_is_alive(device_t dev); /* did probe succeed? */