d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

iscsi: Fetch limits based on a socket rather than assuming global limits.

Browse Source 
cxgbei needs the ability to return different limits based on the
connection (e.g. if the connection is over a T5 adapter or a T6
adapter as well as factoring in the MTU).

This change plumbs through the changes in the ioctls without changing
any of the backends.  The limits callback passed to icl_register now
accepts a second socket argument which holds the integer file
descriptor.  To support ABI compatiblity for old binaries, the
callback should return "global" values if the socket fd is zero.

The CTL_ISCSI_LIMITS argument used with CTL_ISCSI by ctld(8) now
accepts the socket fd in a field that was previously part of a
reserved spare field.  Old binaries zero this request which results in
passing a socket fd of 0 to the limits callback.

The ISCSIDREQUEST ioctl no longer returns limits.  Instead, iscsid(8)
invokes a new ISCSIDLIMITS ioctl after establishing the connection via
connect(2).  For ABI compat, if the old ISCSIDREQUEST is invoked, the
global limits are still fetched (with a socket fd of 0) and returned.

Reviewed by:	mav
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D34928
This commit is contained in:
John Baldwin 2022-04-18 12:48:42 -07:00
parent f0df722733
commit 7b02c1e8c6
13 changed files with 170 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sys/cam/ctl/ctl_frontend_iscsi.c
View File

@ -1896,7 +1896,7 @@ cfiscsi_ioctl_limits(struct ctl_iscsi *ci)
cilp = (struct ctl_iscsi_limits_params *)&(ci->data);
error = icl_limits(cilp->offload, false, &idl);
error = icl_limits(cilp->offload, false, cilp->socket, &idl);
if (error != 0) {
ci->status = CTL_ISCSI_ERROR;
snprintf(ci->error_str, sizeof(ci->error_str),

5
sys/cam/ctl/ctl_ioctl.h
View File

@ -669,9 +669,12 @@ struct ctl_iscsi_terminate_params {
struct ctl_iscsi_limits_params {
/* passed to kernel */
char offload[CTL_ISCSI_OFFLOAD_LEN];
int socket;
/* passed to userland */
size_t spare;
#ifdef __LP64__
int spare;
#endif
int max_recv_data_segment_length;
int max_send_data_segment_length;
int max_burst_length;

2
sys/dev/cxgbe/cxgbei/icl_cxgbei.c
View File

@ -1773,7 +1773,7 @@ cxgbei_limits(struct adapter *sc, void *arg)
}
static int
icl_cxgbei_limits(struct icl_drv_limits *idl)
icl_cxgbei_limits(struct icl_drv_limits *idl, int socket)
{
/* Maximum allowed by the RFC. cxgbei_limits will clip them. */

10
sys/dev/iscsi/icl.c
View File

@ -60,7 +60,8 @@ struct icl_module {
char *im_name;
bool im_iser;
int im_priority;
int (*im_limits)(struct icl_drv_limits *idl);
int (*im_limits)(struct icl_drv_limits *idl,
int socket);
struct icl_conn *(*im_new_conn)(const char *name,
struct mtx *lock);
};
@ -184,7 +185,8 @@ icl_new_conn(const char *offload, bool iser, const char *name, struct mtx *lock)
}
int
icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
icl_limits(const char *offload, bool iser, int socket,
struct icl_drv_limits *idl)
{
struct icl_module *im;
int error;
@ -197,7 +199,7 @@ icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
return (ENXIO);
}
error = im->im_limits(idl);
error = im->im_limits(idl, socket);
sx_sunlock(&sc->sc_lock);
/*
@ -232,7 +234,7 @@ icl_limits(const char *offload, bool iser, struct icl_drv_limits *idl)
int
icl_register(const char *offload, bool iser, int priority,
int (*limits)(struct icl_drv_limits *),
int (*limits)(struct icl_drv_limits *, int),
struct icl_conn *(*new_conn)(const char *, struct mtx *))
{
struct icl_module *im;

4
sys/dev/iscsi/icl.h
View File

@ -137,10 +137,10 @@ typedef void (*icl_pdu_cb)(struct icl_pdu *, int error);
struct icl_conn *icl_new_conn(const char *offload, bool iser, const char *name,
struct mtx *lock);
int icl_limits(const char *offload, bool iser,
int icl_limits(const char *offload, bool iser, int socket,
struct icl_drv_limits *idl);
int icl_register(const char *offload, bool iser, int priority,
int (*limits)(struct icl_drv_limits *),
int (*limits)(struct icl_drv_limits *, int),
struct icl_conn *(*new_conn)(const char *, struct mtx *));
int icl_unregister(const char *offload, bool rdma);

2
sys/dev/iscsi/icl_soft.c
View File

@ -1652,7 +1652,7 @@ icl_soft_conn_transfer_done(struct icl_conn *ic, void *prv)
}
static int
icl_soft_limits(struct icl_drv_limits *idl)
icl_soft_limits(struct icl_drv_limits *idl, int socket)
{
idl->idl_max_recv_data_segment_length = max_data_segment_length;

110
sys/dev/iscsi/iscsi.c
View File

@ -78,6 +78,20 @@ __FBSDID("$FreeBSD$");
FEATURE(iscsi_kernel_proxy, "iSCSI initiator built with ICL_KERNEL_PROXY");
#endif
#ifdef COMPAT_FREEBSD13
struct iscsi_daemon_request13 {
unsigned int idr_session_id;
struct iscsi_session_conf idr_conf;
uint8_t idr_isid[6];
uint16_t idr_tsih;
uint16_t idr_spare_cid;
struct iscsi_session_limits idr_limits;
int idr_spare[4];
};
#define ISCSIDWAIT13 _IOR('I', 0x01, struct iscsi_daemon_request13)
#endif
/*
* XXX: This is global so the iscsi_unload() can access it.
* Think about how to do this properly.
@ -1423,10 +1437,9 @@ iscsi_pdu_handle_reject(struct icl_pdu *response)
static int
iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
struct iscsi_daemon_request *request)
struct iscsi_daemon_request *request, bool freebsd13)
{
struct iscsi_session *is;
struct icl_drv_limits idl;
int error;
sx_slock(&sc->sc_lock);
@ -1472,29 +1485,78 @@ iscsi_ioctl_daemon_wait(struct iscsi_softc *sc,
memcpy(&request->idr_conf, &is->is_conf,
sizeof(request->idr_conf));
error = icl_limits(is->is_conf.isc_offload,
is->is_conf.isc_iser, &idl);
if (error != 0) {
ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
"failed with error %d", is->is_conf.isc_offload,
error);
sx_sunlock(&sc->sc_lock);
return (error);
}
request->idr_limits.isl_max_recv_data_segment_length =
idl.idl_max_recv_data_segment_length;
request->idr_limits.isl_max_send_data_segment_length =
idl.idl_max_send_data_segment_length;
request->idr_limits.isl_max_burst_length =
idl.idl_max_burst_length;
request->idr_limits.isl_first_burst_length =
idl.idl_first_burst_length;
#ifdef COMPAT_FREEBSD13
if (freebsd13) {
struct icl_drv_limits idl;
struct iscsi_daemon_request13 *request13;
error = icl_limits(is->is_conf.isc_offload,
is->is_conf.isc_iser, 0, &idl);
if (error != 0) {
ISCSI_SESSION_WARN(is, "icl_limits for "
"offload \"%s\" failed with error %d",
is->is_conf.isc_offload, error);
sx_sunlock(&sc->sc_lock);
return (error);
}
request13 = (struct iscsi_daemon_request13 *)request;
request13->idr_limits.isl_max_recv_data_segment_length =
idl.idl_max_recv_data_segment_length;
request13->idr_limits.isl_max_send_data_segment_length =
idl.idl_max_send_data_segment_length;
request13->idr_limits.isl_max_burst_length =
idl.idl_max_burst_length;
request13->idr_limits.isl_first_burst_length =
idl.idl_first_burst_length;
}
#endif
sx_sunlock(&sc->sc_lock);
return (0);
}
}
static int
iscsi_ioctl_daemon_limits(struct iscsi_softc *sc,
struct iscsi_daemon_limits *limits)
{
struct icl_drv_limits idl;
struct iscsi_session *is;
int error;
sx_slock(&sc->sc_lock);
/*
* Find the session to fetch limits for.
*/
TAILQ_FOREACH(is, &sc->sc_sessions, is_next) {
if (is->is_id == limits->idl_session_id)
break;
}
if (is == NULL) {
sx_sunlock(&sc->sc_lock);
return (ESRCH);
}
error = icl_limits(is->is_conf.isc_offload, is->is_conf.isc_iser,
limits->idl_socket, &idl);
sx_sunlock(&sc->sc_lock);
if (error != 0) {
ISCSI_SESSION_WARN(is, "icl_limits for offload \"%s\" "
"failed with error %d", is->is_conf.isc_offload, error);
return (error);
}
limits->idl_limits.isl_max_recv_data_segment_length =
idl.idl_max_recv_data_segment_length;
limits->idl_limits.isl_max_send_data_segment_length =
idl.idl_max_send_data_segment_length;
limits->idl_limits.isl_max_burst_length =
idl.idl_max_burst_length;
limits->idl_limits.isl_first_burst_length =
idl.idl_first_burst_length;
return (0);
}
static int
iscsi_ioctl_daemon_handoff(struct iscsi_softc *sc,
struct iscsi_daemon_handoff *handoff)
@ -2179,7 +2241,15 @@ iscsi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int mode,
switch (cmd) {
case ISCSIDWAIT:
return (iscsi_ioctl_daemon_wait(sc,
(struct iscsi_daemon_request *)arg));
(struct iscsi_daemon_request *)arg, false));
#ifdef COMPAT_FREEBSD13
case ISCSIDWAIT13:
return (iscsi_ioctl_daemon_wait(sc,
(struct iscsi_daemon_request *)arg, true));
#endif
case ISCSIDLIMITS:
return (iscsi_ioctl_daemon_limits(sc,
(struct iscsi_daemon_limits *)arg));
case ISCSIDHANDOFF:
return (iscsi_ioctl_daemon_handoff(sc,
(struct iscsi_daemon_handoff *)arg));

8
sys/dev/iscsi/iscsi_ioctl.h
View File

@ -117,10 +117,15 @@ struct iscsi_daemon_request {
uint8_t idr_isid[6];
uint16_t idr_tsih;
uint16_t idr_spare_cid;
struct iscsi_session_limits idr_limits;
int idr_spare[4];
};
struct iscsi_daemon_limits {
unsigned int idl_session_id;
int idl_socket;
struct iscsi_session_limits idl_limits;
};
struct iscsi_daemon_handoff {
unsigned int idh_session_id;
int idh_socket;
@ -150,6 +155,7 @@ struct iscsi_daemon_fail {
#define ISCSIDWAIT _IOR('I', 0x01, struct iscsi_daemon_request)
#define ISCSIDHANDOFF _IOW('I', 0x02, struct iscsi_daemon_handoff)
#define ISCSIDFAIL _IOW('I', 0x03, struct iscsi_daemon_fail)
#define ISCSIDLIMITS _IOWR('I', 0x07, struct iscsi_daemon_limits)
#ifdef ICL_KERNEL_PROXY

2
sys/dev/iser/icl_iser.c
View File

@ -511,7 +511,7 @@ iser_conn_task_done(struct icl_conn *ic, void *prv)
}
static int
iser_limits(struct icl_drv_limits *idl)
iser_limits(struct icl_drv_limits *idl, int socket)
{
idl->idl_max_recv_data_segment_length = 128 * 1024;

2
usr.sbin/ctld/ctld.h
View File

@ -356,7 +356,7 @@ int kernel_lun_add(struct lun *lun);
int kernel_lun_modify(struct lun *lun);
int kernel_lun_remove(struct lun *lun);
void kernel_handoff(struct ctld_connection *conn);
void kernel_limits(const char *offload,
void kernel_limits(const char *offload, int s,
int *max_recv_data_segment_length,
int *max_send_data_segment_length,
int *max_burst_length,

3
usr.sbin/ctld/kernel.c
View File

@ -954,7 +954,7 @@ kernel_handoff(struct ctld_connection *conn)
}
void
kernel_limits(const char *offload, int *max_recv_dsl, int *max_send_dsl,
kernel_limits(const char *offload, int s, int *max_recv_dsl, int *max_send_dsl,
int *max_burst_length, int *first_burst_length)
{
struct ctl_iscsi req;
@ -967,6 +967,7 @@ kernel_limits(const char *offload, int *max_recv_dsl, int *max_send_dsl,
if (offload != NULL) {
strlcpy(cilp->offload, offload, sizeof(cilp->offload));
}
cilp->socket = s;
if (ioctl(ctl_fd, CTL_ISCSI, &req) == -1) {
log_err(1, "error issuing CTL_ISCSI ioctl; "

1
usr.sbin/ctld/login.c
View File

@ -698,6 +698,7 @@ login_negotiate(struct ctld_connection *conn, struct pdu *request)
conn->conn_max_burst_limit = (1 << 24) - 1;
conn->conn_first_burst_limit = (1 << 24) - 1;
kernel_limits(conn->conn_portal->p_portal_group->pg_offload,
conn->conn.conn_socket,
&conn->conn_max_recv_data_segment_limit,
&conn->conn_max_send_data_segment_limit,
&conn->conn_max_burst_limit,

87
usr.sbin/iscsid/iscsid.c
View File

@ -225,7 +225,6 @@ static struct iscsid_connection *
connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
{
struct iscsid_connection *conn;
struct iscsi_session_limits *isl;
struct addrinfo *from_ai, *to_ai;
const char *from_addr, *to_addr;
#ifdef ICL_KERNEL_PROXY
@ -249,38 +248,6 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
sizeof(conn->conn.conn_isid));
conn->conn.conn_tsih = request->idr_tsih;
/*
* Read the driver limits and provide reasonable defaults for the ones
* the driver doesn't care about. If a max_snd_dsl is not explicitly
* provided by the driver then we'll make sure both conn->max_snd_dsl
* and isl->max_snd_dsl are set to the rcv_dsl. This preserves historic
* behavior.
*/
isl = &conn->conn_limits;
memcpy(isl, &request->idr_limits, sizeof(*isl));
if (isl->isl_max_recv_data_segment_length == 0)
isl->isl_max_recv_data_segment_length = (1 << 24) - 1;
if (isl->isl_max_send_data_segment_length == 0)
isl->isl_max_send_data_segment_length =
isl->isl_max_recv_data_segment_length;
if (isl->isl_max_burst_length == 0)
isl->isl_max_burst_length = (1 << 24) - 1;
if (isl->isl_first_burst_length == 0)
isl->isl_first_burst_length = (1 << 24) - 1;
if (isl->isl_first_burst_length > isl->isl_max_burst_length)
isl->isl_first_burst_length = isl->isl_max_burst_length;
/*
* Limit default send length in case it won't be negotiated.
* We can't do it for other limits, since they may affect both
* sender and receiver operation, and we must obey defaults.
*/
if (conn->conn.conn_max_send_data_segment_length >
isl->isl_max_send_data_segment_length) {
conn->conn.conn_max_send_data_segment_length =
isl->isl_max_send_data_segment_length;
}
from_addr = conn->conn_conf.isc_initiator_addr;
to_addr = conn->conn_conf.isc_target_addr;
@ -436,6 +403,56 @@ connection_new(int iscsi_fd, const struct iscsi_daemon_request *request)
return (conn);
}
static void
limits(struct iscsid_connection *conn)
{
struct iscsi_daemon_limits idl;
struct iscsi_session_limits *isl;
int error;
log_debugx("fetching limits from the kernel");
memset(&idl, 0, sizeof(idl));
idl.idl_session_id = conn->conn_session_id;
idl.idl_socket = conn->conn.conn_socket;
error = ioctl(conn->conn_iscsi_fd, ISCSIDLIMITS, &idl);
if (error != 0)
log_err(1, "ISCSIDLIMITS");
/*
* Read the driver limits and provide reasonable defaults for the ones
* the driver doesn't care about. If a max_snd_dsl is not explicitly
* provided by the driver then we'll make sure both conn->max_snd_dsl
* and isl->max_snd_dsl are set to the rcv_dsl. This preserves historic
* behavior.
*/
isl = &conn->conn_limits;
memcpy(isl, &idl.idl_limits, sizeof(*isl));
if (isl->isl_max_recv_data_segment_length == 0)
isl->isl_max_recv_data_segment_length = (1 << 24) - 1;
if (isl->isl_max_send_data_segment_length == 0)
isl->isl_max_send_data_segment_length =
isl->isl_max_recv_data_segment_length;
if (isl->isl_max_burst_length == 0)
isl->isl_max_burst_length = (1 << 24) - 1;
if (isl->isl_first_burst_length == 0)
isl->isl_first_burst_length = (1 << 24) - 1;
if (isl->isl_first_burst_length > isl->isl_max_burst_length)
isl->isl_first_burst_length = isl->isl_max_burst_length;
/*
* Limit default send length in case it won't be negotiated.
* We can't do it for other limits, since they may affect both
* sender and receiver operation, and we must obey defaults.
*/
if (conn->conn.conn_max_send_data_segment_length >
isl->isl_max_send_data_segment_length) {
conn->conn.conn_max_send_data_segment_length =
isl->isl_max_send_data_segment_length;
}
}
static void
handoff(struct iscsid_connection *conn)
{
@ -502,6 +519,7 @@ capsicate(struct iscsid_connection *conn)
ISCSIDSEND,
ISCSIDRECEIVE,
#endif
ISCSIDLIMITS,
ISCSIDHANDOFF,
ISCSIDFAIL,
ISCSISADD,
@ -626,8 +644,9 @@ handle_request(int iscsi_fd, const struct iscsi_daemon_request *request, int tim
}
conn = connection_new(iscsi_fd, request);
set_timeout(timeout);
capsicate(conn);
limits(conn);
set_timeout(timeout);
login(conn);
if (conn->conn_conf.isc_discovery != 0)
discovery(conn);