d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MFC r278239 and r278374.

Browse Source 
r278239:
cxgbe(4): reserve id for iSCSI upper layer driver.

r278374:
cxgbe(4): tidy up some of the interaction between the Upper Layer
Drivers (ULDs) and the base if_cxgbe driver.

Track the per-adapter activation of ULDs in a new "active_ulds" field.
This was done pretty arbitrarily before this change -- via TOM_INIT_DONE
in adapter->flags for TOM, and the (1 << MAX_NPORTS) bit in
adapter->offload_map for iWARP.

iWARP and hw-accelerated iSCSI rely on the TOE (supported by the TOM
ULD).  The rules are:
a) If the iWARP and/or iSCSI ULDs are available when TOE is enabled then
   iWARP and/or iSCSI are enabled too.
b) When the iWARP and iSCSI modules are loaded they go looking for
   adapters with TOE enabled and enable themselves on that adapter.
c) You cannot deactivate or unload the TOM module from underneath iWARP
   or iSCSI.  Any such attempt will fail with EBUSY.
This commit is contained in:
np 2015-06-06 18:00:36 +00:00
parent fcae6374c4
commit edfdc5c75d
6 changed files with 60 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
sys/dev/cxgbe/adapter.h
View File

@ -186,7 +186,7 @@ enum {
/* INTR_DIRECT = (1 << 2), No longer used. */
MASTER_PF = (1 << 3),
ADAP_SYSCTL_CTX = (1 << 4),
TOM_INIT_DONE = (1 << 5),
/* TOM_INIT_DONE= (1 << 5), No longer used */
BUF_PACKING_OK = (1 << 6),
CXGBE_BUSY = (1 << 9),
@ -751,7 +751,8 @@ struct adapter {
uint16_t doorbells;
int open_device_map;
#ifdef TCP_OFFLOAD
int offload_map;
int offload_map; /* ports with IFCAP_TOE enabled */
int active_ulds; /* ULDs activated on this adapter */
#endif
int flags;

13
sys/dev/cxgbe/iw_cxgbe/device.c
View File

@ -213,7 +213,7 @@ c4iw_activate(struct adapter *sc)
ASSERT_SYNCHRONIZED_OP(sc);
if (isset(&sc->offload_map, MAX_NPORTS)) {
if (uld_active(sc, ULD_IWARP)) {
KASSERT(0, ("%s: RDMA already eanbled on sc %p", __func__, sc));
return (0);
}
@ -265,9 +265,9 @@ c4iw_activate_all(struct adapter *sc, void *arg __unused)
if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwact") != 0)
return;
if (!isset(&sc->offload_map, MAX_NPORTS) &&
t4_activate_uld(sc, ULD_IWARP) == 0)
setbit(&sc->offload_map, MAX_NPORTS);
/* Activate iWARP if any port on this adapter has IFCAP_TOE enabled. */
if (sc->offload_map && !uld_active(sc, ULD_IWARP))
(void) t4_activate_uld(sc, ULD_IWARP);
end_synchronized_op(sc, 0);
}
@ -279,9 +279,8 @@ c4iw_deactivate_all(struct adapter *sc, void *arg __unused)
if (begin_synchronized_op(sc, NULL, SLEEP_OK | INTR_OK, "t4iwdea") != 0)
return;
if (isset(&sc->offload_map, MAX_NPORTS) &&
t4_deactivate_uld(sc, ULD_IWARP) == 0)
clrbit(&sc->offload_map, MAX_NPORTS);
if (uld_active(sc, ULD_IWARP))
(void) t4_deactivate_uld(sc, ULD_IWARP);
end_synchronized_op(sc, 0);
}

7
sys/dev/cxgbe/offload.h
View File

@ -127,8 +127,10 @@ struct t4_virt_res { /* virtualized HW resources */
#ifdef TCP_OFFLOAD
enum {
ULD_TOM = 1,
ULD_IWARP = 2,
ULD_TOM = 0,
ULD_IWARP,
ULD_ISCSI,
ULD_MAX = ULD_ISCSI
};
struct adapter;
@ -155,5 +157,6 @@ int t4_unregister_uld(struct uld_info *);
int t4_activate_uld(struct adapter *, int);
int t4_deactivate_uld(struct adapter *, int);
void t4_iscsi_init(struct ifnet *, unsigned int, const unsigned int *);
int uld_active(struct adapter *, int);
#endif
#endif

53
sys/dev/cxgbe/t4_main.c
View File

@ -7057,7 +7057,7 @@ set_filter_mode(struct adapter *sc, uint32_t mode)
}
#ifdef TCP_OFFLOAD
if (sc->offload_map) {
if (uld_active(sc, ULD_TOM)) {
rc = EBUSY;
goto done;
}
@ -8142,7 +8142,7 @@ toe_capability(struct port_info *pi, int enable)
if (isset(&sc->offload_map, pi->port_id))
return (0);
if (!(sc->flags & TOM_INIT_DONE)) {
if (!uld_active(sc, ULD_TOM)) {
rc = t4_activate_uld(sc, ULD_TOM);
if (rc == EAGAIN) {
log(LOG_WARNING,
@ -8153,16 +8153,22 @@ toe_capability(struct port_info *pi, int enable)
return (rc);
KASSERT(sc->tom_softc != NULL,
("%s: TOM activated but softc NULL", __func__));
KASSERT(sc->flags & TOM_INIT_DONE,
KASSERT(uld_active(sc, ULD_TOM),
("%s: TOM activated but flag not set", __func__));
}
/* Activate iWARP and iSCSI too, if the modules are loaded. */
if (!uld_active(sc, ULD_IWARP))
(void) t4_activate_uld(sc, ULD_IWARP);
if (!uld_active(sc, ULD_ISCSI))
(void) t4_activate_uld(sc, ULD_ISCSI);
setbit(&sc->offload_map, pi->port_id);
} else {
if (!isset(&sc->offload_map, pi->port_id))
return (0);
KASSERT(sc->flags & TOM_INIT_DONE,
KASSERT(uld_active(sc, ULD_TOM),
("%s: TOM never initialized?", __func__));
clrbit(&sc->offload_map, pi->port_id);
}
@ -8222,11 +8228,15 @@ t4_unregister_uld(struct uld_info *ui)
int
t4_activate_uld(struct adapter *sc, int id)
{
int rc = EAGAIN;
int rc;
struct uld_info *ui;
ASSERT_SYNCHRONIZED_OP(sc);
if (id < 0 || id > ULD_MAX)
return (EINVAL);
rc = EAGAIN; /* kldoad the module with this ULD and try again. */
sx_slock(&t4_uld_list_lock);
SLIST_FOREACH(ui, &t4_uld_list, link) {
@ -8234,16 +8244,18 @@ t4_activate_uld(struct adapter *sc, int id)
if (!(sc->flags & FULL_INIT_DONE)) {
rc = adapter_full_init(sc);
if (rc != 0)
goto done;
break;
}
rc = ui->activate(sc);
if (rc == 0)
if (rc == 0) {
setbit(&sc->active_ulds, id);
ui->refcount++;
goto done;
}
break;
}
}
done:
sx_sunlock(&t4_uld_list_lock);
return (rc);
@ -8252,26 +8264,41 @@ t4_activate_uld(struct adapter *sc, int id)
int
t4_deactivate_uld(struct adapter *sc, int id)
{
int rc = EINVAL;
int rc;
struct uld_info *ui;
ASSERT_SYNCHRONIZED_OP(sc);
if (id < 0 || id > ULD_MAX)
return (EINVAL);
rc = ENXIO;
sx_slock(&t4_uld_list_lock);
SLIST_FOREACH(ui, &t4_uld_list, link) {
if (ui->uld_id == id) {
rc = ui->deactivate(sc);
if (rc == 0)
if (rc == 0) {
clrbit(&sc->active_ulds, id);
ui->refcount--;
goto done;
}
break;
}
}
done:
sx_sunlock(&t4_uld_list_lock);
return (rc);
}
int
uld_active(struct adapter *sc, int uld_id)
{
MPASS(uld_id >= 0 && uld_id <= ULD_MAX);
return (isset(&sc->active_ulds, uld_id));
}
#endif
/*

2
sys/dev/cxgbe/tom/t4_listen.c
View File

@ -523,7 +523,7 @@ t4_listen_start(struct toedev *tod, struct tcpcb *tp)
goto done;
}
KASSERT(sc->flags & TOM_INIT_DONE,
KASSERT(uld_active(sc, ULD_TOM),
("%s: TOM not initialized", __func__));
#endif

9
sys/dev/cxgbe/tom/t4_tom.c
View File

@ -745,7 +745,7 @@ update_clip(struct adapter *sc, void *arg __unused)
if (begin_synchronized_op(sc, NULL, HOLD_LOCK, "t4tomuc"))
return;
if (sc->flags & TOM_INIT_DONE)
if (uld_active(sc, ULD_TOM))
update_clip_table(sc, sc->tom_softc);
end_synchronized_op(sc, LOCK_HELD);
@ -1024,7 +1024,6 @@ t4_tom_activate(struct adapter *sc)
TOEDEV(sc->port[i]->ifp) = &td->tod;
sc->tom_softc = td;
sc->flags |= TOM_INIT_DONE;
register_toedev(sc->tom_softc);
done:
@ -1047,6 +1046,9 @@ t4_tom_deactivate(struct adapter *sc)
if (sc->offload_map != 0)
return (EBUSY); /* at least one port has IFCAP_TOE enabled */
if (uld_active(sc, ULD_IWARP) || uld_active(sc, ULD_ISCSI))
return (EBUSY); /* both iWARP and iSCSI rely on the TOE. */
mtx_lock(&td->toep_list_lock);
if (!TAILQ_EMPTY(&td->toep_list))
rc = EBUSY;
@ -1067,7 +1069,6 @@ t4_tom_deactivate(struct adapter *sc)
unregister_toedev(sc->tom_softc);
free_tom_data(sc, td);
sc->tom_softc = NULL;
sc->flags &= ~TOM_INIT_DONE;
}
return (rc);
@ -1121,7 +1122,7 @@ tom_uninit(struct adapter *sc, void *arg __unused)
return;
/* Try to free resources (works only if no port has IFCAP_TOE) */
if (sc->flags & TOM_INIT_DONE)
if (uld_active(sc, ULD_TOM))
t4_deactivate_uld(sc, ULD_TOM);
end_synchronized_op(sc, 0);