Bring in the Mellanox implementation of iSER (iSCSI over RDMA) initiator,
written by Sagi Grimberg <sagig at mellanox.com> and Max Gurtovoy <maxg at mellanox.com>. This code comes from https://github.com/sagigrimberg/iser-freebsd, branch iser-rebase-11-current-r291993. It's not connected to the build just yet; it still needs some tweaks to adapt to my changes to iSCSI infrastructure. Big thanks to Mellanox for their support for FreeBSD! Obtained from: Mellanox Technologies MFC after: 1 month Relnotes: yes
This commit is contained in:
parent
f159d7d6f0
commit
4814a0a4ce
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=300723
582
sys/dev/iser/icl_iser.c
Normal file
582
sys/dev/iser/icl_iser.c
Normal file
@ -0,0 +1,582 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "icl_iser.h"
|
||||
|
||||
SYSCTL_NODE(_kern, OID_AUTO, iser, CTLFLAG_RW, 0, "iSER module");
|
||||
int iser_debug = 0;
|
||||
SYSCTL_INT(_kern_iser, OID_AUTO, debug, CTLFLAG_RWTUN,
|
||||
&iser_debug, 0, "Enable iser debug messages");
|
||||
|
||||
static MALLOC_DEFINE(M_ICL_ISER, "icl_iser", "iSCSI iser backend");
|
||||
static uma_zone_t icl_pdu_zone;
|
||||
|
||||
static volatile u_int icl_iser_ncons;
|
||||
struct iser_global ig;
|
||||
|
||||
static icl_conn_new_pdu_t iser_conn_new_pdu;
|
||||
static icl_conn_pdu_free_t iser_conn_pdu_free;
|
||||
static icl_conn_pdu_data_segment_length_t iser_conn_pdu_data_segment_length;
|
||||
static icl_conn_pdu_append_data_t iser_conn_pdu_append_data;
|
||||
static icl_conn_pdu_queue_t iser_conn_pdu_queue;
|
||||
static icl_conn_handoff_t iser_conn_handoff;
|
||||
static icl_conn_free_t iser_conn_free;
|
||||
static icl_conn_close_t iser_conn_close;
|
||||
static icl_conn_release_t iser_conn_release;
|
||||
static icl_conn_connect_t iser_conn_connect;
|
||||
static icl_conn_connected_t iser_conn_connected;
|
||||
static icl_conn_task_setup_t iser_conn_task_setup;
|
||||
static icl_conn_task_done_t iser_conn_task_done;
|
||||
static icl_conn_pdu_get_data_t iser_conn_pdu_get_data;
|
||||
|
||||
static kobj_method_t icl_iser_methods[] = {
|
||||
KOBJMETHOD(icl_conn_new_pdu, iser_conn_new_pdu),
|
||||
KOBJMETHOD(icl_conn_pdu_free, iser_conn_pdu_free),
|
||||
KOBJMETHOD(icl_conn_pdu_data_segment_length, iser_conn_pdu_data_segment_length),
|
||||
KOBJMETHOD(icl_conn_pdu_append_data, iser_conn_pdu_append_data),
|
||||
KOBJMETHOD(icl_conn_pdu_queue, iser_conn_pdu_queue),
|
||||
KOBJMETHOD(icl_conn_handoff, iser_conn_handoff),
|
||||
KOBJMETHOD(icl_conn_free, iser_conn_free),
|
||||
KOBJMETHOD(icl_conn_close, iser_conn_close),
|
||||
KOBJMETHOD(icl_conn_release, iser_conn_release),
|
||||
KOBJMETHOD(icl_conn_connect, iser_conn_connect),
|
||||
KOBJMETHOD(icl_conn_connected, iser_conn_connected),
|
||||
KOBJMETHOD(icl_conn_task_setup, iser_conn_task_setup),
|
||||
KOBJMETHOD(icl_conn_task_done, iser_conn_task_done),
|
||||
KOBJMETHOD(icl_conn_pdu_get_data, iser_conn_pdu_get_data),
|
||||
{ 0, 0 }
|
||||
};
|
||||
|
||||
DEFINE_CLASS(icl_iser, icl_iser_methods, sizeof(struct iser_conn));
|
||||
|
||||
/**
|
||||
* iser_initialize_headers() - Initialize task headers
|
||||
* @pdu: iser pdu
|
||||
* @iser_conn: iser connection
|
||||
*
|
||||
* Notes:
|
||||
* This routine may race with iser teardown flow for scsi
|
||||
* error handling TMFs. So for TMF we should acquire the
|
||||
* state mutex to avoid dereferencing the IB device which
|
||||
* may have already been terminated (racing teardown sequence).
|
||||
*/
|
||||
int
|
||||
iser_initialize_headers(struct icl_iser_pdu *pdu, struct iser_conn *iser_conn)
|
||||
{
|
||||
struct iser_tx_desc *tx_desc = &pdu->desc;
|
||||
struct iser_device *device = iser_conn->ib_conn.device;
|
||||
u64 dma_addr;
|
||||
int ret = 0;
|
||||
|
||||
dma_addr = ib_dma_map_single(device->ib_device, (void *)tx_desc,
|
||||
ISER_HEADERS_LEN, DMA_TO_DEVICE);
|
||||
if (ib_dma_mapping_error(device->ib_device, dma_addr)) {
|
||||
ret = -ENOMEM;
|
||||
goto out;
|
||||
}
|
||||
|
||||
tx_desc->mapped = true;
|
||||
tx_desc->dma_addr = dma_addr;
|
||||
tx_desc->tx_sg[0].addr = tx_desc->dma_addr;
|
||||
tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
|
||||
tx_desc->tx_sg[0].lkey = device->mr->lkey;
|
||||
|
||||
out:
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
int
|
||||
iser_conn_pdu_append_data(struct icl_conn *ic, struct icl_pdu *request,
|
||||
const void *addr, size_t len, int flags)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
|
||||
if (request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_LOGIN_REQUEST ||
|
||||
request->ip_bhs->bhs_opcode & ISCSI_BHS_OPCODE_TEXT_REQUEST) {
|
||||
ISER_DBG("copy to login buff");
|
||||
memcpy(iser_conn->login_req_buf, addr, len);
|
||||
request->ip_data_len = len;
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_pdu_get_data(struct icl_conn *ic, struct icl_pdu *ip,
|
||||
size_t off, void *addr, size_t len)
|
||||
{
|
||||
/* If we have a receive data, copy it to upper layer buffer */
|
||||
if (ip->ip_data_mbuf)
|
||||
memcpy(addr, ip->ip_data_mbuf + off, len);
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate icl_pdu with empty BHS to fill up by the caller.
|
||||
*/
|
||||
struct icl_pdu *
|
||||
iser_new_pdu(struct icl_conn *ic, int flags)
|
||||
{
|
||||
struct icl_iser_pdu *iser_pdu;
|
||||
struct icl_pdu *ip;
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
|
||||
iser_pdu = uma_zalloc(icl_pdu_zone, flags | M_ZERO);
|
||||
if (iser_pdu == NULL) {
|
||||
ISER_WARN("failed to allocate %zd bytes", sizeof(*iser_pdu));
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
iser_pdu->iser_conn = iser_conn;
|
||||
ip = &iser_pdu->icl_pdu;
|
||||
ip->ip_conn = ic;
|
||||
ip->ip_bhs = &iser_pdu->desc.iscsi_header;
|
||||
|
||||
return (ip);
|
||||
}
|
||||
|
||||
struct icl_pdu *
|
||||
iser_conn_new_pdu(struct icl_conn *ic, int flags)
|
||||
{
|
||||
return (iser_new_pdu(ic, flags));
|
||||
}
|
||||
|
||||
void
|
||||
iser_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
|
||||
{
|
||||
struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
|
||||
|
||||
uma_zfree(icl_pdu_zone, iser_pdu);
|
||||
}
|
||||
|
||||
size_t
|
||||
iser_conn_pdu_data_segment_length(struct icl_conn *ic,
|
||||
const struct icl_pdu *request)
|
||||
{
|
||||
uint32_t len = 0;
|
||||
|
||||
len += request->ip_bhs->bhs_data_segment_len[0];
|
||||
len <<= 8;
|
||||
len += request->ip_bhs->bhs_data_segment_len[1];
|
||||
len <<= 8;
|
||||
len += request->ip_bhs->bhs_data_segment_len[2];
|
||||
|
||||
return (len);
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_pdu_free(struct icl_conn *ic, struct icl_pdu *ip)
|
||||
{
|
||||
iser_pdu_free(ic, ip);
|
||||
}
|
||||
|
||||
static bool
|
||||
is_control_opcode(uint8_t opcode)
|
||||
{
|
||||
bool is_control = false;
|
||||
|
||||
switch (opcode & ISCSI_OPCODE_MASK) {
|
||||
case ISCSI_BHS_OPCODE_NOP_OUT:
|
||||
case ISCSI_BHS_OPCODE_LOGIN_REQUEST:
|
||||
case ISCSI_BHS_OPCODE_LOGOUT_REQUEST:
|
||||
case ISCSI_BHS_OPCODE_TEXT_REQUEST:
|
||||
is_control = true;
|
||||
break;
|
||||
case ISCSI_BHS_OPCODE_SCSI_COMMAND:
|
||||
is_control = false;
|
||||
break;
|
||||
default:
|
||||
ISER_ERR("unknown opcode %d", opcode);
|
||||
}
|
||||
|
||||
return (is_control);
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_pdu_queue(struct icl_conn *ic, struct icl_pdu *ip)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
|
||||
int ret;
|
||||
|
||||
ret = iser_initialize_headers(iser_pdu, iser_conn);
|
||||
if (ret) {
|
||||
ISER_ERR("Failed to map TX descriptor pdu %p", iser_pdu);
|
||||
return;
|
||||
}
|
||||
|
||||
if (is_control_opcode(ip->ip_bhs->bhs_opcode)) {
|
||||
ret = iser_send_control(iser_conn, iser_pdu);
|
||||
if (unlikely(ret))
|
||||
ISER_ERR("Failed to send control pdu %p", iser_pdu);
|
||||
} else {
|
||||
ret = iser_send_command(iser_conn, iser_pdu);
|
||||
if (unlikely(ret))
|
||||
ISER_ERR("Failed to send command pdu %p", iser_pdu);
|
||||
}
|
||||
}
|
||||
|
||||
static struct icl_conn *
|
||||
iser_new_conn(const char *name, struct mtx *lock)
|
||||
{
|
||||
struct iser_conn *iser_conn;
|
||||
struct icl_conn *ic;
|
||||
|
||||
refcount_acquire(&icl_iser_ncons);
|
||||
|
||||
iser_conn = (struct iser_conn *)kobj_create(&icl_iser_class, M_ICL_ISER, M_WAITOK | M_ZERO);
|
||||
if (!iser_conn) {
|
||||
ISER_ERR("failed to allocate iser conn");
|
||||
refcount_release(&icl_iser_ncons);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
cv_init(&iser_conn->up_cv, "iser_cv");
|
||||
sx_init(&iser_conn->state_mutex, "iser_conn_state_mutex");
|
||||
mtx_init(&iser_conn->ib_conn.beacon.flush_lock, "flush_lock", NULL, MTX_DEF);
|
||||
cv_init(&iser_conn->ib_conn.beacon.flush_cv, "flush_cv");
|
||||
mtx_init(&iser_conn->ib_conn.lock, "lock", NULL, MTX_DEF);
|
||||
|
||||
ic = &iser_conn->icl_conn;
|
||||
ic->ic_lock = lock;
|
||||
ic->ic_name = name;
|
||||
ic->ic_driver = strdup("iser", M_TEMP);
|
||||
ic->ic_iser = true;
|
||||
|
||||
return (ic);
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_free(struct icl_conn *ic)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
|
||||
cv_destroy(&iser_conn->ib_conn.beacon.flush_cv);
|
||||
mtx_destroy(&iser_conn->ib_conn.beacon.flush_lock);
|
||||
sx_destroy(&iser_conn->state_mutex);
|
||||
cv_destroy(&iser_conn->up_cv);
|
||||
kobj_delete((struct kobj *)iser_conn, M_ICL_ISER);
|
||||
refcount_release(&icl_iser_ncons);
|
||||
}
|
||||
|
||||
int
|
||||
iser_conn_handoff(struct icl_conn *ic, int cmds_max)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
int error = 0;
|
||||
|
||||
sx_xlock(&iser_conn->state_mutex);
|
||||
if (iser_conn->state != ISER_CONN_UP) {
|
||||
error = EINVAL;
|
||||
ISER_ERR("iser_conn %p state is %d, teardown started\n",
|
||||
iser_conn, iser_conn->state);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/*
|
||||
* In discovery session no need to allocate rx desc and posting recv
|
||||
* work request
|
||||
*/
|
||||
if (ic->ic_session_type_discovery(ic))
|
||||
goto out;
|
||||
|
||||
error = iser_alloc_rx_descriptors(iser_conn, cmds_max);
|
||||
if (error)
|
||||
goto out;
|
||||
|
||||
error = iser_post_recvm(iser_conn, iser_conn->min_posted_rx);
|
||||
if (error)
|
||||
goto post_error;
|
||||
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
return (error);
|
||||
|
||||
post_error:
|
||||
iser_free_rx_descriptors(iser_conn);
|
||||
out:
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
return (error);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Frees all conn objects
|
||||
*/
|
||||
void
|
||||
iser_conn_release(struct icl_conn *ic)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct iser_conn *curr, *tmp;
|
||||
|
||||
mtx_lock(&ig.connlist_mutex);
|
||||
/*
|
||||
* Search for iser connection in global list.
|
||||
* It may not be there in case of failure in connection establishment
|
||||
* stage.
|
||||
*/
|
||||
list_for_each_entry_safe(curr, tmp, &ig.connlist, conn_list) {
|
||||
if (iser_conn == curr) {
|
||||
ISER_WARN("found iser_conn %p", iser_conn);
|
||||
list_del(&iser_conn->conn_list);
|
||||
}
|
||||
}
|
||||
mtx_unlock(&ig.connlist_mutex);
|
||||
|
||||
/*
|
||||
* In case we reconnecting or removing session, we need to
|
||||
* release IB resources (which is safe to call more than once).
|
||||
*/
|
||||
sx_xlock(&iser_conn->state_mutex);
|
||||
iser_free_ib_conn_res(iser_conn, true);
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
|
||||
if (ib_conn->cma_id != NULL) {
|
||||
rdma_destroy_id(ib_conn->cma_id);
|
||||
ib_conn->cma_id = NULL;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_close(struct icl_conn *ic)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
|
||||
ISER_INFO("closing conn %p", iser_conn);
|
||||
|
||||
sx_xlock(&iser_conn->state_mutex);
|
||||
/*
|
||||
* In case iser connection is waiting on conditional variable
|
||||
* (state PENDING) and we try to close it before connection establishment,
|
||||
* we need to signal it to continue releasing connection properly.
|
||||
*/
|
||||
if (!iser_conn_terminate(iser_conn) && iser_conn->state == ISER_CONN_PENDING)
|
||||
cv_signal(&iser_conn->up_cv);
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
|
||||
}
|
||||
|
||||
int
|
||||
iser_conn_connect(struct icl_conn *ic, int domain, int socktype,
|
||||
int protocol, struct sockaddr *from_sa, struct sockaddr *to_sa)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
int err = 0;
|
||||
|
||||
sx_xlock(&iser_conn->state_mutex);
|
||||
/* the device is known only --after-- address resolution */
|
||||
ib_conn->device = NULL;
|
||||
|
||||
iser_conn->state = ISER_CONN_PENDING;
|
||||
|
||||
ib_conn->cma_id = rdma_create_id(iser_cma_handler, (void *)iser_conn,
|
||||
RDMA_PS_TCP, IB_QPT_RC);
|
||||
if (IS_ERR(ib_conn->cma_id)) {
|
||||
err = -PTR_ERR(ib_conn->cma_id);
|
||||
ISER_ERR("rdma_create_id failed: %d", err);
|
||||
goto id_failure;
|
||||
}
|
||||
|
||||
err = rdma_resolve_addr(ib_conn->cma_id, from_sa, to_sa, 1000);
|
||||
if (err) {
|
||||
ISER_ERR("rdma_resolve_addr failed: %d", err);
|
||||
if (err < 0)
|
||||
err = -err;
|
||||
goto addr_failure;
|
||||
}
|
||||
|
||||
ISER_DBG("before cv_wait: %p", iser_conn);
|
||||
cv_wait(&iser_conn->up_cv, &iser_conn->state_mutex);
|
||||
ISER_DBG("after cv_wait: %p", iser_conn);
|
||||
|
||||
if (iser_conn->state != ISER_CONN_UP) {
|
||||
err = EIO;
|
||||
goto addr_failure;
|
||||
}
|
||||
|
||||
err = iser_alloc_login_buf(iser_conn);
|
||||
if (err)
|
||||
goto addr_failure;
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
|
||||
mtx_lock(&ig.connlist_mutex);
|
||||
list_add(&iser_conn->conn_list, &ig.connlist);
|
||||
mtx_unlock(&ig.connlist_mutex);
|
||||
|
||||
return (0);
|
||||
|
||||
id_failure:
|
||||
ib_conn->cma_id = NULL;
|
||||
addr_failure:
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
return (err);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with session spinlock held.
|
||||
* No need to lock state mutex on an advisory check.
|
||||
**/
|
||||
bool
|
||||
iser_conn_connected(struct icl_conn *ic)
|
||||
{
|
||||
struct iser_conn *iser_conn = icl_to_iser_conn(ic);
|
||||
|
||||
return (iser_conn->state == ISER_CONN_UP);
|
||||
}
|
||||
|
||||
int
|
||||
iser_conn_task_setup(struct icl_conn *ic, struct ccb_scsiio *csio,
|
||||
uint32_t *task_tagp, void **prvp, struct icl_pdu *ip)
|
||||
{
|
||||
struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
|
||||
|
||||
*prvp = ip;
|
||||
iser_pdu->csio = csio;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
void
|
||||
iser_conn_task_done(struct icl_conn *ic, void *prv)
|
||||
{
|
||||
struct icl_pdu *ip = prv;
|
||||
struct icl_iser_pdu *iser_pdu = icl_to_iser_pdu(ip);
|
||||
struct iser_device *device = iser_pdu->iser_conn->ib_conn.device;
|
||||
struct iser_tx_desc *tx_desc = &iser_pdu->desc;
|
||||
|
||||
if (iser_pdu->dir[ISER_DIR_IN]) {
|
||||
iser_unreg_rdma_mem(iser_pdu, ISER_DIR_IN);
|
||||
iser_dma_unmap_task_data(iser_pdu,
|
||||
&iser_pdu->data[ISER_DIR_IN],
|
||||
DMA_FROM_DEVICE);
|
||||
}
|
||||
|
||||
if (iser_pdu->dir[ISER_DIR_OUT]) {
|
||||
iser_unreg_rdma_mem(iser_pdu, ISER_DIR_OUT);
|
||||
iser_dma_unmap_task_data(iser_pdu,
|
||||
&iser_pdu->data[ISER_DIR_OUT],
|
||||
DMA_TO_DEVICE);
|
||||
}
|
||||
|
||||
if (likely(tx_desc->mapped)) {
|
||||
ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr,
|
||||
ISER_HEADERS_LEN, DMA_TO_DEVICE);
|
||||
tx_desc->mapped = false;
|
||||
}
|
||||
|
||||
iser_pdu_free(ic, ip);
|
||||
}
|
||||
|
||||
static u_int32_t
|
||||
iser_hba_misc()
|
||||
{
|
||||
return (PIM_UNMAPPED);
|
||||
}
|
||||
|
||||
static int
|
||||
iser_limits(size_t *limitp)
|
||||
{
|
||||
*limitp = 128 * 1024;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
icl_iser_load(void)
|
||||
{
|
||||
int error;
|
||||
|
||||
ISER_DBG("Starting iSER datamover...");
|
||||
|
||||
icl_pdu_zone = uma_zcreate("icl_iser_pdu", sizeof(struct icl_iser_pdu),
|
||||
NULL, NULL, NULL, NULL,
|
||||
UMA_ALIGN_PTR, 0);
|
||||
/* FIXME: Check rc */
|
||||
|
||||
refcount_init(&icl_iser_ncons, 0);
|
||||
|
||||
error = icl_register("iser", 0, iser_limits, iser_new_conn, iser_hba_misc);
|
||||
KASSERT(error == 0, ("failed to register iser"));
|
||||
|
||||
memset(&ig, 0, sizeof(struct iser_global));
|
||||
|
||||
/* device init is called only after the first addr resolution */
|
||||
sx_init(&ig.device_list_mutex, "global_device_lock");
|
||||
INIT_LIST_HEAD(&ig.device_list);
|
||||
mtx_init(&ig.connlist_mutex, "global_conn_lock", NULL, MTX_DEF);
|
||||
INIT_LIST_HEAD(&ig.connlist);
|
||||
sx_init(&ig.close_conns_mutex, "global_close_conns_lock");
|
||||
|
||||
return (error);
|
||||
}
|
||||
|
||||
static int
|
||||
icl_iser_unload(void)
|
||||
{
|
||||
ISER_DBG("Removing iSER datamover...");
|
||||
|
||||
if (icl_iser_ncons != 0)
|
||||
return (EBUSY);
|
||||
|
||||
sx_destroy(&ig.close_conns_mutex);
|
||||
mtx_destroy(&ig.connlist_mutex);
|
||||
sx_destroy(&ig.device_list_mutex);
|
||||
|
||||
icl_unregister("iser");
|
||||
|
||||
uma_zdestroy(icl_pdu_zone);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
icl_iser_modevent(module_t mod, int what, void *arg)
|
||||
{
|
||||
switch (what) {
|
||||
case MOD_LOAD:
|
||||
return (icl_iser_load());
|
||||
case MOD_UNLOAD:
|
||||
return (icl_iser_unload());
|
||||
default:
|
||||
return (EINVAL);
|
||||
}
|
||||
}
|
||||
|
||||
moduledata_t icl_iser_data = {
|
||||
.name = "icl_iser",
|
||||
.evhand = icl_iser_modevent,
|
||||
.priv = 0
|
||||
};
|
||||
|
||||
DECLARE_MODULE(icl_iser, icl_iser_data, SI_SUB_DRIVERS, SI_ORDER_MIDDLE);
|
||||
MODULE_DEPEND(icl_iser, icl, 1, 1, 1);
|
||||
MODULE_DEPEND(icl_iser, iscsi, 1, 1, 1);
|
||||
MODULE_DEPEND(icl_iser, ibcore, 1, 1, 1);
|
||||
MODULE_DEPEND(icl_iser, linuxkpi, 1, 1, 1);
|
||||
MODULE_VERSION(icl_iser, 1);
|
||||
|
547
sys/dev/iser/icl_iser.h
Normal file
547
sys/dev/iser/icl_iser.h
Normal file
@ -0,0 +1,547 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef ICL_ISER_H
|
||||
#define ICL_ISER_H
|
||||
|
||||
/*
|
||||
* iSCSI Common Layer for RDMA.
|
||||
*/
|
||||
|
||||
#include <sys/cdefs.h>
|
||||
#include <sys/param.h>
|
||||
#include <sys/capsicum.h>
|
||||
#include <sys/condvar.h>
|
||||
#include <sys/conf.h>
|
||||
#include <sys/file.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/kthread.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/mbuf.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/module.h>
|
||||
#include <sys/protosw.h>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/socketvar.h>
|
||||
#include <sys/sysctl.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/sx.h>
|
||||
#include <sys/uio.h>
|
||||
#include <sys/taskqueue.h>
|
||||
#include <sys/bio.h>
|
||||
#include <vm/uma.h>
|
||||
#include <netinet/in.h>
|
||||
#include <netinet/tcp.h>
|
||||
#include <dev/iscsi/icl.h>
|
||||
#include <dev/iscsi/iscsi_proto.h>
|
||||
#include <icl_conn_if.h>
|
||||
#include <cam/cam.h>
|
||||
#include <cam/cam_ccb.h>
|
||||
#include <rdma/ib_verbs.h>
|
||||
#include <rdma/ib_fmr_pool.h>
|
||||
#include <rdma/rdma_cm.h>
|
||||
|
||||
|
||||
#define ISER_DBG(X, ...) \
|
||||
do { \
|
||||
if (unlikely(iser_debug > 2)) \
|
||||
printf("DEBUG: %s: " X "\n", \
|
||||
__func__, ## __VA_ARGS__); \
|
||||
} while (0)
|
||||
|
||||
#define ISER_INFO(X, ...) \
|
||||
do { \
|
||||
if (unlikely(iser_debug > 1)) \
|
||||
printf("INFO: %s: " X "\n", \
|
||||
__func__, ## __VA_ARGS__); \
|
||||
} while (0)
|
||||
|
||||
#define ISER_WARN(X, ...) \
|
||||
do { \
|
||||
if (unlikely(iser_debug > 0)) { \
|
||||
printf("WARNING: %s: " X "\n", \
|
||||
__func__, ## __VA_ARGS__); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ISER_ERR(X, ...) \
|
||||
printf("ERROR: %s: " X "\n", __func__, ## __VA_ARGS__)
|
||||
|
||||
#define ISER_VER 0x10
|
||||
#define ISER_WSV 0x08
|
||||
#define ISER_RSV 0x04
|
||||
|
||||
#define ISER_FASTREG_LI_WRID 0xffffffffffffffffULL
|
||||
#define ISER_BEACON_WRID 0xfffffffffffffffeULL
|
||||
|
||||
#define SHIFT_4K 12
|
||||
#define SIZE_4K (1ULL << SHIFT_4K)
|
||||
#define MASK_4K (~(SIZE_4K-1))
|
||||
|
||||
/* support up to 512KB in one RDMA */
|
||||
#define ISCSI_ISER_SG_TABLESIZE (0x80000 >> SHIFT_4K)
|
||||
#define ISER_DEF_XMIT_CMDS_MAX 256
|
||||
|
||||
/* the max RX (recv) WR supported by the iSER QP is defined by *
|
||||
* max_recv_wr = commands_max + recv_beacon */
|
||||
#define ISER_QP_MAX_RECV_DTOS (ISER_DEF_XMIT_CMDS_MAX + 1)
|
||||
#define ISER_MIN_POSTED_RX (ISER_DEF_XMIT_CMDS_MAX >> 2)
|
||||
|
||||
/* QP settings */
|
||||
/* Maximal bounds on received asynchronous PDUs */
|
||||
#define ISER_MAX_RX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */
|
||||
#define ISER_MAX_TX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), SCSI_TMFUNC(2), LOGOUT(1) */
|
||||
|
||||
/* the max TX (send) WR supported by the iSER QP is defined by *
|
||||
* max_send_wr = T * (1 + D) + C ; D is how many inflight dataouts we expect *
|
||||
* to have at max for SCSI command. The tx posting & completion handling code *
|
||||
* supports -EAGAIN scheme where tx is suspended till the QP has room for more *
|
||||
* send WR. D=8 comes from 64K/8K */
|
||||
|
||||
#define ISER_INFLIGHT_DATAOUTS 8
|
||||
|
||||
/* the send_beacon increase the max_send_wr by 1 */
|
||||
#define ISER_QP_MAX_REQ_DTOS (ISER_DEF_XMIT_CMDS_MAX * \
|
||||
(1 + ISER_INFLIGHT_DATAOUTS) + \
|
||||
ISER_MAX_TX_MISC_PDUS + \
|
||||
ISER_MAX_RX_MISC_PDUS + 1)
|
||||
|
||||
#define ISER_GET_MAX_XMIT_CMDS(send_wr) ((send_wr \
|
||||
- ISER_MAX_TX_MISC_PDUS \
|
||||
- ISER_MAX_RX_MISC_PDUS - 1) / \
|
||||
(1 + ISER_INFLIGHT_DATAOUTS))
|
||||
|
||||
#define ISER_WC_BATCH_COUNT 16
|
||||
#define ISER_SIGNAL_CMD_COUNT 32
|
||||
|
||||
/* Maximal QP's recommended per CQ. In case we use more QP's per CQ we might *
|
||||
* encounter a CQ overrun state. */
|
||||
#define ISCSI_ISER_MAX_CONN 8
|
||||
#define ISER_MAX_RX_LEN (ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
|
||||
#define ISER_MAX_TX_LEN (ISER_QP_MAX_REQ_DTOS * ISCSI_ISER_MAX_CONN)
|
||||
#define ISER_MAX_CQ_LEN (ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
|
||||
ISCSI_ISER_MAX_CONN)
|
||||
|
||||
#define ISER_ZBVA_NOT_SUPPORTED 0x80
|
||||
#define ISER_SEND_W_INV_NOT_SUPPORTED 0x40
|
||||
|
||||
#define icl_to_iser_conn(ic) \
|
||||
container_of(ic, struct iser_conn, icl_conn)
|
||||
#define icl_to_iser_pdu(ip) \
|
||||
container_of(ip, struct icl_iser_pdu, icl_pdu)
|
||||
|
||||
/**
|
||||
* struct iser_hdr - iSER header
|
||||
*
|
||||
* @flags: flags support (zbva, remote_inv)
|
||||
* @rsvd: reserved
|
||||
* @write_stag: write rkey
|
||||
* @write_va: write virtual address
|
||||
* @reaf_stag: read rkey
|
||||
* @read_va: read virtual address
|
||||
*/
|
||||
struct iser_hdr {
|
||||
u8 flags;
|
||||
u8 rsvd[3];
|
||||
__be32 write_stag;
|
||||
__be64 write_va;
|
||||
__be32 read_stag;
|
||||
__be64 read_va;
|
||||
} __attribute__((packed));
|
||||
|
||||
struct iser_cm_hdr {
|
||||
u8 flags;
|
||||
u8 rsvd[3];
|
||||
} __packed;
|
||||
|
||||
/* Constant PDU lengths calculations */
|
||||
#define ISER_HEADERS_LEN (sizeof(struct iser_hdr) + ISCSI_BHS_SIZE)
|
||||
|
||||
#define ISER_RECV_DATA_SEG_LEN 128
|
||||
#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN)
|
||||
|
||||
#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN)
|
||||
|
||||
enum iser_conn_state {
|
||||
ISER_CONN_INIT, /* descriptor allocd, no conn */
|
||||
ISER_CONN_PENDING, /* in the process of being established */
|
||||
ISER_CONN_UP, /* up and running */
|
||||
ISER_CONN_TERMINATING, /* in the process of being terminated */
|
||||
ISER_CONN_DOWN, /* shut down */
|
||||
ISER_CONN_STATES_NUM
|
||||
};
|
||||
|
||||
enum iser_task_status {
|
||||
ISER_TASK_STATUS_INIT = 0,
|
||||
ISER_TASK_STATUS_STARTED,
|
||||
ISER_TASK_STATUS_COMPLETED
|
||||
};
|
||||
|
||||
enum iser_data_dir {
|
||||
ISER_DIR_IN = 0, /* to initiator */
|
||||
ISER_DIR_OUT, /* from initiator */
|
||||
ISER_DIRS_NUM
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_mem_reg - iSER memory registration info
|
||||
*
|
||||
* @sge: memory region sg element
|
||||
* @rkey: memory region remote key
|
||||
* @mem_h: pointer to registration context (FMR/Fastreg)
|
||||
*/
|
||||
struct iser_mem_reg {
|
||||
struct ib_sge sge;
|
||||
u32 rkey;
|
||||
void *mem_h;
|
||||
};
|
||||
|
||||
enum iser_desc_type {
|
||||
ISCSI_TX_CONTROL ,
|
||||
ISCSI_TX_SCSI_COMMAND,
|
||||
ISCSI_TX_DATAOUT
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_data_buf - iSER data buffer
|
||||
*
|
||||
* @sg: pointer to the sg list
|
||||
* @size: num entries of this sg
|
||||
* @data_len: total beffer byte len
|
||||
* @dma_nents: returned by dma_map_sg
|
||||
* @copy_buf: allocated copy buf for SGs unaligned
|
||||
* for rdma which are copied
|
||||
* @orig_sg: pointer to the original sg list (in case
|
||||
* we used a copy)
|
||||
* @sg_single: SG-ified clone of a non SG SC or
|
||||
* unaligned SG
|
||||
*/
|
||||
struct iser_data_buf {
|
||||
struct scatterlist sgl[ISCSI_ISER_SG_TABLESIZE];
|
||||
void *sg;
|
||||
unsigned int size;
|
||||
unsigned long data_len;
|
||||
unsigned int dma_nents;
|
||||
char *copy_buf;
|
||||
struct scatterlist *orig_sg;
|
||||
struct scatterlist sg_single;
|
||||
};
|
||||
|
||||
/* fwd declarations */
|
||||
struct iser_conn;
|
||||
struct ib_conn;
|
||||
struct iser_device;
|
||||
|
||||
/**
|
||||
* struct iser_tx_desc - iSER TX descriptor (for send wr_id)
|
||||
*
|
||||
* @iser_header: iser header
|
||||
* @iscsi_header: iscsi header (bhs)
|
||||
* @type: command/control/dataout
|
||||
* @dma_addr: header buffer dma_address
|
||||
* @tx_sg: sg[0] points to iser/iscsi headers
|
||||
* sg[1] optionally points to either of immediate data
|
||||
* unsolicited data-out or control
|
||||
* @num_sge: number sges used on this TX task
|
||||
* @mapped: indicates if the descriptor is dma mapped
|
||||
*/
|
||||
struct iser_tx_desc {
|
||||
struct iser_hdr iser_header;
|
||||
struct iscsi_bhs iscsi_header __attribute__((packed));
|
||||
enum iser_desc_type type;
|
||||
u64 dma_addr;
|
||||
struct ib_sge tx_sg[2];
|
||||
int num_sge;
|
||||
bool mapped;
|
||||
};
|
||||
|
||||
#define ISER_RX_PAD_SIZE (256 - (ISER_RX_PAYLOAD_SIZE + \
|
||||
sizeof(u64) + sizeof(struct ib_sge)))
|
||||
/**
|
||||
* struct iser_rx_desc - iSER RX descriptor (for recv wr_id)
|
||||
*
|
||||
* @iser_header: iser header
|
||||
* @iscsi_header: iscsi header
|
||||
* @data: received data segment
|
||||
* @dma_addr: receive buffer dma address
|
||||
* @rx_sg: ib_sge of receive buffer
|
||||
* @pad: for sense data TODO: Modify to maximum sense length supported
|
||||
*/
|
||||
struct iser_rx_desc {
|
||||
struct iser_hdr iser_header;
|
||||
struct iscsi_bhs iscsi_header;
|
||||
char data[ISER_RECV_DATA_SEG_LEN];
|
||||
u64 dma_addr;
|
||||
struct ib_sge rx_sg;
|
||||
char pad[ISER_RX_PAD_SIZE];
|
||||
} __attribute__((packed));
|
||||
|
||||
struct icl_iser_pdu {
|
||||
struct icl_pdu icl_pdu;
|
||||
struct iser_tx_desc desc;
|
||||
struct iser_conn *iser_conn;
|
||||
enum iser_task_status status;
|
||||
struct ccb_scsiio *csio;
|
||||
int command_sent;
|
||||
int dir[ISER_DIRS_NUM];
|
||||
struct iser_mem_reg rdma_reg[ISER_DIRS_NUM];
|
||||
struct iser_data_buf data[ISER_DIRS_NUM];
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_comp - iSER completion context
|
||||
*
|
||||
* @device: pointer to device handle
|
||||
* @cq: completion queue
|
||||
* @wcs: work completion array
|
||||
* @tq: taskqueue handle
|
||||
* @task: task to run task_fn
|
||||
* @active_qps: Number of active QPs attached
|
||||
* to completion context
|
||||
*/
|
||||
struct iser_comp {
|
||||
struct iser_device *device;
|
||||
struct ib_cq *cq;
|
||||
struct ib_wc wcs[ISER_WC_BATCH_COUNT];
|
||||
struct taskqueue *tq;
|
||||
struct task task;
|
||||
int active_qps;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_device - iSER device handle
|
||||
*
|
||||
* @ib_device: RDMA device
|
||||
* @pd: Protection Domain for this device
|
||||
* @dev_attr: Device attributes container
|
||||
* @mr: Global DMA memory region
|
||||
* @event_handler: IB events handle routine
|
||||
* @ig_list: entry in devices list
|
||||
* @refcount: Reference counter, dominated by open iser connections
|
||||
* @comps_used: Number of completion contexts used, Min between online
|
||||
* cpus and device max completion vectors
|
||||
* @comps: Dinamically allocated array of completion handlers
|
||||
*/
|
||||
struct iser_device {
|
||||
struct ib_device *ib_device;
|
||||
struct ib_pd *pd;
|
||||
struct ib_device_attr dev_attr;
|
||||
struct ib_mr *mr;
|
||||
struct ib_event_handler event_handler;
|
||||
struct list_head ig_list;
|
||||
int refcount;
|
||||
int comps_used;
|
||||
struct iser_comp *comps;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_reg_resources - Fast registration recources
|
||||
*
|
||||
* @mr: memory region
|
||||
* @frpl: fast reg page list
|
||||
* @mr_valid: is mr valid indicator
|
||||
*/
|
||||
struct iser_reg_resources {
|
||||
struct ib_mr *mr;
|
||||
struct ib_fast_reg_page_list *frpl;
|
||||
u8 mr_valid:1;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct fast_reg_descriptor - Fast registration descriptor
|
||||
*
|
||||
* @list: entry in connection fastreg pool
|
||||
* @rsc: data buffer registration resources
|
||||
*/
|
||||
struct fast_reg_descriptor {
|
||||
struct list_head list;
|
||||
struct iser_reg_resources rsc;
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* struct iser_beacon - beacon to signal all flush errors were drained
|
||||
*
|
||||
* @send: send wr
|
||||
* @recv: recv wr
|
||||
* @flush_lock: protects flush_cv
|
||||
* @flush_cv: condition variable for beacon flush
|
||||
*/
|
||||
struct iser_beacon {
|
||||
union {
|
||||
struct ib_send_wr send;
|
||||
struct ib_recv_wr recv;
|
||||
};
|
||||
struct mtx flush_lock;
|
||||
struct cv flush_cv;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct ib_conn - Infiniband related objects
|
||||
*
|
||||
* @cma_id: rdma_cm connection maneger handle
|
||||
* @qp: Connection Queue-pair
|
||||
* @device: reference to iser device
|
||||
* @comp: iser completion context
|
||||
*/
|
||||
struct ib_conn {
|
||||
struct rdma_cm_id *cma_id;
|
||||
struct ib_qp *qp;
|
||||
int post_recv_buf_count;
|
||||
u8 sig_count;
|
||||
struct ib_recv_wr rx_wr[ISER_MIN_POSTED_RX];
|
||||
struct iser_device *device;
|
||||
struct iser_comp *comp;
|
||||
struct iser_beacon beacon;
|
||||
struct mtx lock;
|
||||
union {
|
||||
struct {
|
||||
struct ib_fmr_pool *pool;
|
||||
struct iser_page_vec *page_vec;
|
||||
} fmr;
|
||||
struct {
|
||||
struct list_head pool;
|
||||
int pool_size;
|
||||
} fastreg;
|
||||
};
|
||||
};
|
||||
|
||||
struct iser_conn {
|
||||
struct icl_conn icl_conn;
|
||||
struct ib_conn ib_conn;
|
||||
struct cv up_cv;
|
||||
struct list_head conn_list;
|
||||
struct sx state_mutex;
|
||||
enum iser_conn_state state;
|
||||
int qp_max_recv_dtos;
|
||||
int min_posted_rx;
|
||||
u16 max_cmds;
|
||||
char *login_buf;
|
||||
char *login_req_buf, *login_resp_buf;
|
||||
u64 login_req_dma, login_resp_dma;
|
||||
unsigned int rx_desc_head;
|
||||
struct iser_rx_desc *rx_descs;
|
||||
u32 num_rx_descs;
|
||||
};
|
||||
|
||||
/**
|
||||
* struct iser_global: iSER global context
|
||||
*
|
||||
* @device_list_mutex: protects device_list
|
||||
* @device_list: iser devices global list
|
||||
* @connlist_mutex: protects connlist
|
||||
* @connlist: iser connections global list
|
||||
* @desc_cache: kmem cache for tx dataout
|
||||
* @close_conns_mutex: serializes conns closure
|
||||
*/
|
||||
struct iser_global {
|
||||
struct sx device_list_mutex;
|
||||
struct list_head device_list;
|
||||
struct mtx connlist_mutex;
|
||||
struct list_head connlist;
|
||||
struct sx close_conns_mutex;
|
||||
};
|
||||
|
||||
extern struct iser_global ig;
|
||||
extern int iser_debug;
|
||||
|
||||
void
|
||||
iser_create_send_desc(struct iser_conn *, struct iser_tx_desc *);
|
||||
|
||||
int
|
||||
iser_post_recvl(struct iser_conn *);
|
||||
|
||||
int
|
||||
iser_post_recvm(struct iser_conn *, int);
|
||||
|
||||
int
|
||||
iser_alloc_login_buf(struct iser_conn *iser_conn);
|
||||
|
||||
void
|
||||
iser_free_login_buf(struct iser_conn *iser_conn);
|
||||
|
||||
int
|
||||
iser_post_send(struct ib_conn *, struct iser_tx_desc *, bool);
|
||||
|
||||
void
|
||||
iser_snd_completion(struct iser_tx_desc *, struct ib_conn *);
|
||||
|
||||
void
|
||||
iser_rcv_completion(struct iser_rx_desc *, unsigned long,
|
||||
struct ib_conn *);
|
||||
|
||||
void
|
||||
iser_pdu_free(struct icl_conn *, struct icl_pdu *);
|
||||
|
||||
struct icl_pdu *
|
||||
iser_new_pdu(struct icl_conn *ic, int flags);
|
||||
|
||||
int
|
||||
iser_alloc_rx_descriptors(struct iser_conn *, int);
|
||||
|
||||
void
|
||||
iser_free_rx_descriptors(struct iser_conn *);
|
||||
|
||||
int
|
||||
iser_initialize_headers(struct icl_iser_pdu *, struct iser_conn *);
|
||||
|
||||
int
|
||||
iser_send_control(struct iser_conn *, struct icl_iser_pdu *);
|
||||
|
||||
int
|
||||
iser_send_command(struct iser_conn *, struct icl_iser_pdu *);
|
||||
|
||||
int
|
||||
iser_reg_rdma_mem(struct icl_iser_pdu *, enum iser_data_dir);
|
||||
|
||||
void
|
||||
iser_unreg_rdma_mem(struct icl_iser_pdu *, enum iser_data_dir);
|
||||
|
||||
int
|
||||
iser_create_fastreg_pool(struct ib_conn *, unsigned);
|
||||
|
||||
void
|
||||
iser_free_fastreg_pool(struct ib_conn *);
|
||||
|
||||
int
|
||||
iser_dma_map_task_data(struct icl_iser_pdu *,
|
||||
struct iser_data_buf *, enum iser_data_dir,
|
||||
enum dma_data_direction);
|
||||
|
||||
int
|
||||
iser_conn_terminate(struct iser_conn *);
|
||||
|
||||
void
|
||||
iser_free_ib_conn_res(struct iser_conn *, bool);
|
||||
|
||||
void
|
||||
iser_dma_unmap_task_data(struct icl_iser_pdu *, struct iser_data_buf *,
|
||||
enum dma_data_direction);
|
||||
|
||||
int
|
||||
iser_cma_handler(struct rdma_cm_id *, struct rdma_cm_event *);
|
||||
|
||||
#endif /* !ICL_ISER_H */
|
539
sys/dev/iser/iser_initiator.c
Normal file
539
sys/dev/iser/iser_initiator.c
Normal file
@ -0,0 +1,539 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "icl_iser.h"
|
||||
|
||||
static MALLOC_DEFINE(M_ISER_INITIATOR, "iser_initiator", "iser initiator backend");
|
||||
|
||||
/* Register user buffer memory and initialize passive rdma
|
||||
* dto descriptor. Data size is stored in
|
||||
* task->data[ISER_DIR_IN].data_len, Protection size
|
||||
* os stored in task->prot[ISER_DIR_IN].data_len
|
||||
*/
|
||||
static int
|
||||
iser_prepare_read_cmd(struct icl_iser_pdu *iser_pdu)
|
||||
{
|
||||
struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
|
||||
struct iser_data_buf *buf_in = &iser_pdu->data[ISER_DIR_IN];
|
||||
struct iser_mem_reg *mem_reg;
|
||||
int err;
|
||||
|
||||
err = iser_dma_map_task_data(iser_pdu,
|
||||
buf_in,
|
||||
ISER_DIR_IN,
|
||||
DMA_FROM_DEVICE);
|
||||
if (err)
|
||||
return (err);
|
||||
|
||||
err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_IN);
|
||||
if (err) {
|
||||
ISER_ERR("Failed to set up Data-IN RDMA");
|
||||
return (err);
|
||||
}
|
||||
|
||||
mem_reg = &iser_pdu->rdma_reg[ISER_DIR_IN];
|
||||
|
||||
hdr->flags |= ISER_RSV;
|
||||
hdr->read_stag = cpu_to_be32(mem_reg->rkey);
|
||||
hdr->read_va = cpu_to_be64(mem_reg->sge.addr);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
/* Register user buffer memory and initialize passive rdma
|
||||
* dto descriptor. Data size is stored in
|
||||
* task->data[ISER_DIR_OUT].data_len, Protection size
|
||||
* is stored at task->prot[ISER_DIR_OUT].data_len
|
||||
*/
|
||||
static int
|
||||
iser_prepare_write_cmd(struct icl_iser_pdu *iser_pdu)
|
||||
{
|
||||
struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
|
||||
struct iser_data_buf *buf_out = &iser_pdu->data[ISER_DIR_OUT];
|
||||
struct iser_mem_reg *mem_reg;
|
||||
int err;
|
||||
|
||||
err = iser_dma_map_task_data(iser_pdu,
|
||||
buf_out,
|
||||
ISER_DIR_OUT,
|
||||
DMA_TO_DEVICE);
|
||||
if (err)
|
||||
return (err);
|
||||
|
||||
err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_OUT);
|
||||
if (err) {
|
||||
ISER_ERR("Failed to set up Data-out RDMA");
|
||||
return (err);
|
||||
}
|
||||
|
||||
mem_reg = &iser_pdu->rdma_reg[ISER_DIR_OUT];
|
||||
|
||||
hdr->flags |= ISER_WSV;
|
||||
hdr->write_stag = cpu_to_be32(mem_reg->rkey);
|
||||
hdr->write_va = cpu_to_be64(mem_reg->sge.addr);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
/* creates a new tx descriptor and adds header regd buffer */
|
||||
void
|
||||
iser_create_send_desc(struct iser_conn *iser_conn,
|
||||
struct iser_tx_desc *tx_desc)
|
||||
{
|
||||
struct iser_device *device = iser_conn->ib_conn.device;
|
||||
|
||||
ib_dma_sync_single_for_cpu(device->ib_device,
|
||||
tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
|
||||
|
||||
memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
|
||||
tx_desc->iser_header.flags = ISER_VER;
|
||||
|
||||
tx_desc->num_sge = 1;
|
||||
|
||||
if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
|
||||
tx_desc->tx_sg[0].lkey = device->mr->lkey;
|
||||
ISER_DBG("sdesc %p lkey mismatch, fixing", tx_desc);
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
iser_free_login_buf(struct iser_conn *iser_conn)
|
||||
{
|
||||
struct iser_device *device = iser_conn->ib_conn.device;
|
||||
|
||||
if (!iser_conn->login_buf)
|
||||
return;
|
||||
|
||||
if (iser_conn->login_req_dma)
|
||||
ib_dma_unmap_single(device->ib_device,
|
||||
iser_conn->login_req_dma,
|
||||
ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
|
||||
|
||||
if (iser_conn->login_resp_dma)
|
||||
ib_dma_unmap_single(device->ib_device,
|
||||
iser_conn->login_resp_dma,
|
||||
ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
|
||||
|
||||
free(iser_conn->login_buf, M_ISER_INITIATOR);
|
||||
|
||||
/* make sure we never redo any unmapping */
|
||||
iser_conn->login_req_dma = 0;
|
||||
iser_conn->login_resp_dma = 0;
|
||||
iser_conn->login_buf = NULL;
|
||||
}
|
||||
|
||||
int
|
||||
iser_alloc_login_buf(struct iser_conn *iser_conn)
|
||||
{
|
||||
struct iser_device *device = iser_conn->ib_conn.device;
|
||||
int req_err, resp_err;
|
||||
|
||||
BUG_ON(device == NULL);
|
||||
|
||||
iser_conn->login_buf = malloc(ISCSI_DEF_MAX_RECV_SEG_LEN + ISER_RX_LOGIN_SIZE,
|
||||
M_ISER_INITIATOR, M_WAITOK | M_ZERO);
|
||||
|
||||
if (!iser_conn->login_buf)
|
||||
goto out_err;
|
||||
|
||||
iser_conn->login_req_buf = iser_conn->login_buf;
|
||||
iser_conn->login_resp_buf = iser_conn->login_buf +
|
||||
ISCSI_DEF_MAX_RECV_SEG_LEN;
|
||||
|
||||
iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
|
||||
iser_conn->login_req_buf,
|
||||
ISCSI_DEF_MAX_RECV_SEG_LEN,
|
||||
DMA_TO_DEVICE);
|
||||
|
||||
iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
|
||||
iser_conn->login_resp_buf,
|
||||
ISER_RX_LOGIN_SIZE,
|
||||
DMA_FROM_DEVICE);
|
||||
|
||||
req_err = ib_dma_mapping_error(device->ib_device,
|
||||
iser_conn->login_req_dma);
|
||||
resp_err = ib_dma_mapping_error(device->ib_device,
|
||||
iser_conn->login_resp_dma);
|
||||
|
||||
if (req_err || resp_err) {
|
||||
if (req_err)
|
||||
iser_conn->login_req_dma = 0;
|
||||
if (resp_err)
|
||||
iser_conn->login_resp_dma = 0;
|
||||
goto free_login_buf;
|
||||
}
|
||||
|
||||
return (0);
|
||||
|
||||
free_login_buf:
|
||||
iser_free_login_buf(iser_conn);
|
||||
|
||||
out_err:
|
||||
ISER_DBG("unable to alloc or map login buf");
|
||||
return (ENOMEM);
|
||||
}
|
||||
|
||||
int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, int cmds_max)
|
||||
{
|
||||
int i, j;
|
||||
u64 dma_addr;
|
||||
struct iser_rx_desc *rx_desc;
|
||||
struct ib_sge *rx_sg;
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
|
||||
iser_conn->qp_max_recv_dtos = cmds_max;
|
||||
iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
|
||||
|
||||
if (iser_create_fastreg_pool(ib_conn, cmds_max))
|
||||
goto create_rdma_reg_res_failed;
|
||||
|
||||
|
||||
iser_conn->num_rx_descs = cmds_max;
|
||||
iser_conn->rx_descs = malloc(iser_conn->num_rx_descs *
|
||||
sizeof(struct iser_rx_desc), M_ISER_INITIATOR,
|
||||
M_WAITOK | M_ZERO);
|
||||
if (!iser_conn->rx_descs)
|
||||
goto rx_desc_alloc_fail;
|
||||
|
||||
rx_desc = iser_conn->rx_descs;
|
||||
|
||||
for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) {
|
||||
dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
|
||||
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
||||
if (ib_dma_mapping_error(device->ib_device, dma_addr))
|
||||
goto rx_desc_dma_map_failed;
|
||||
|
||||
rx_desc->dma_addr = dma_addr;
|
||||
|
||||
rx_sg = &rx_desc->rx_sg;
|
||||
rx_sg->addr = rx_desc->dma_addr;
|
||||
rx_sg->length = ISER_RX_PAYLOAD_SIZE;
|
||||
rx_sg->lkey = device->mr->lkey;
|
||||
}
|
||||
|
||||
iser_conn->rx_desc_head = 0;
|
||||
|
||||
return (0);
|
||||
|
||||
rx_desc_dma_map_failed:
|
||||
rx_desc = iser_conn->rx_descs;
|
||||
for (j = 0; j < i; j++, rx_desc++)
|
||||
ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
|
||||
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
||||
free(iser_conn->rx_descs, M_ISER_INITIATOR);
|
||||
iser_conn->rx_descs = NULL;
|
||||
rx_desc_alloc_fail:
|
||||
iser_free_fastreg_pool(ib_conn);
|
||||
create_rdma_reg_res_failed:
|
||||
ISER_ERR("failed allocating rx descriptors / data buffers");
|
||||
|
||||
return (ENOMEM);
|
||||
}
|
||||
|
||||
void
|
||||
iser_free_rx_descriptors(struct iser_conn *iser_conn)
|
||||
{
|
||||
int i;
|
||||
struct iser_rx_desc *rx_desc;
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
|
||||
iser_free_fastreg_pool(ib_conn);
|
||||
|
||||
rx_desc = iser_conn->rx_descs;
|
||||
for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
|
||||
ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
|
||||
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
||||
|
||||
free(iser_conn->rx_descs, M_ISER_INITIATOR);
|
||||
|
||||
/* make sure we never redo any unmapping */
|
||||
iser_conn->rx_descs = NULL;
|
||||
}
|
||||
|
||||
static void
|
||||
iser_buf_to_sg(void *buf, struct iser_data_buf *data_buf)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
int i;
|
||||
size_t len, tlen;
|
||||
int offset;
|
||||
|
||||
tlen = data_buf->data_len;
|
||||
|
||||
for (i = 0; 0 < tlen; i++, tlen -= len) {
|
||||
sg = &data_buf->sgl[i];
|
||||
offset = ((uintptr_t)buf) & ~PAGE_MASK;
|
||||
len = min(PAGE_SIZE - offset, tlen);
|
||||
sg_set_buf(sg, buf, len);
|
||||
buf = (void *)(((u64)buf) + (u64)len);
|
||||
}
|
||||
|
||||
data_buf->size = i;
|
||||
sg_mark_end(sg);
|
||||
}
|
||||
|
||||
|
||||
static void
|
||||
iser_bio_to_sg(struct bio *bp, struct iser_data_buf *data_buf)
|
||||
{
|
||||
struct scatterlist *sg;
|
||||
int i;
|
||||
size_t len, tlen;
|
||||
int offset;
|
||||
|
||||
tlen = bp->bio_bcount;
|
||||
offset = bp->bio_ma_offset;
|
||||
|
||||
for (i = 0; 0 < tlen; i++, tlen -= len) {
|
||||
sg = &data_buf->sgl[i];
|
||||
len = min(PAGE_SIZE - offset, tlen);
|
||||
sg_set_page(sg, bp->bio_ma[i], len, offset);
|
||||
offset = 0;
|
||||
}
|
||||
|
||||
data_buf->size = i;
|
||||
sg_mark_end(sg);
|
||||
}
|
||||
|
||||
static int
|
||||
iser_csio_to_sg(struct ccb_scsiio *csio, struct iser_data_buf *data_buf)
|
||||
{
|
||||
struct ccb_hdr *ccbh;
|
||||
int err = 0;
|
||||
|
||||
ccbh = &csio->ccb_h;
|
||||
switch ((ccbh->flags & CAM_DATA_MASK)) {
|
||||
case CAM_DATA_BIO:
|
||||
iser_bio_to_sg((struct bio *) csio->data_ptr, data_buf);
|
||||
break;
|
||||
case CAM_DATA_VADDR:
|
||||
/*
|
||||
* Support KVA buffers for various scsi commands such as:
|
||||
* - REPORT_LUNS
|
||||
* - MODE_SENSE_6
|
||||
* - INQUIRY
|
||||
* - SERVICE_ACTION_IN.
|
||||
* The data of these commands always mapped into KVA.
|
||||
*/
|
||||
iser_buf_to_sg(csio->data_ptr, data_buf);
|
||||
break;
|
||||
default:
|
||||
ISER_ERR("flags 0x%X unimplemented", ccbh->flags);
|
||||
err = EINVAL;
|
||||
}
|
||||
return (err);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
iser_signal_comp(u8 sig_count)
|
||||
{
|
||||
return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);
|
||||
}
|
||||
|
||||
int
|
||||
iser_send_command(struct iser_conn *iser_conn,
|
||||
struct icl_iser_pdu *iser_pdu)
|
||||
{
|
||||
struct iser_data_buf *data_buf;
|
||||
struct iser_tx_desc *tx_desc = &iser_pdu->desc;
|
||||
struct iscsi_bhs_scsi_command *hdr = (struct iscsi_bhs_scsi_command *) &(iser_pdu->desc.iscsi_header);
|
||||
struct ccb_scsiio *csio = iser_pdu->csio;
|
||||
int err = 0;
|
||||
u8 sig_count = ++iser_conn->ib_conn.sig_count;
|
||||
|
||||
/* build the tx desc regd header and add it to the tx desc dto */
|
||||
tx_desc->type = ISCSI_TX_SCSI_COMMAND;
|
||||
iser_create_send_desc(iser_conn, tx_desc);
|
||||
|
||||
if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
|
||||
data_buf = &iser_pdu->data[ISER_DIR_IN];
|
||||
} else {
|
||||
data_buf = &iser_pdu->data[ISER_DIR_OUT];
|
||||
}
|
||||
|
||||
data_buf->sg = csio->data_ptr;
|
||||
data_buf->data_len = csio->dxfer_len;
|
||||
|
||||
if (likely(csio->dxfer_len)) {
|
||||
err = iser_csio_to_sg(csio, data_buf);
|
||||
if (unlikely(err))
|
||||
goto send_command_error;
|
||||
}
|
||||
|
||||
if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
|
||||
err = iser_prepare_read_cmd(iser_pdu);
|
||||
if (err)
|
||||
goto send_command_error;
|
||||
} else if (hdr->bhssc_flags & BHSSC_FLAGS_W) {
|
||||
err = iser_prepare_write_cmd(iser_pdu);
|
||||
if (err)
|
||||
goto send_command_error;
|
||||
}
|
||||
|
||||
err = iser_post_send(&iser_conn->ib_conn, tx_desc,
|
||||
iser_signal_comp(sig_count));
|
||||
if (!err)
|
||||
return (0);
|
||||
|
||||
send_command_error:
|
||||
ISER_ERR("iser_conn %p itt %u len %u err %d", iser_conn,
|
||||
hdr->bhssc_initiator_task_tag,
|
||||
hdr->bhssc_expected_data_transfer_length,
|
||||
err);
|
||||
return (err);
|
||||
}
|
||||
|
||||
int
|
||||
iser_send_control(struct iser_conn *iser_conn,
|
||||
struct icl_iser_pdu *iser_pdu)
|
||||
{
|
||||
struct iser_tx_desc *mdesc;
|
||||
struct iser_device *device;
|
||||
size_t datalen = iser_pdu->icl_pdu.ip_data_len;
|
||||
struct icl_conn *ic = &iser_conn->icl_conn;
|
||||
int err;
|
||||
|
||||
mdesc = &iser_pdu->desc;
|
||||
|
||||
/* build the tx desc regd header and add it to the tx desc dto */
|
||||
mdesc->type = ISCSI_TX_CONTROL;
|
||||
iser_create_send_desc(iser_conn, mdesc);
|
||||
|
||||
device = iser_conn->ib_conn.device;
|
||||
|
||||
if (datalen > 0) {
|
||||
struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
|
||||
ib_dma_sync_single_for_cpu(device->ib_device,
|
||||
iser_conn->login_req_dma, datalen,
|
||||
DMA_TO_DEVICE);
|
||||
|
||||
ib_dma_sync_single_for_device(device->ib_device,
|
||||
iser_conn->login_req_dma, datalen,
|
||||
DMA_TO_DEVICE);
|
||||
|
||||
tx_dsg->addr = iser_conn->login_req_dma;
|
||||
tx_dsg->length = datalen;
|
||||
tx_dsg->lkey = device->mr->lkey;
|
||||
mdesc->num_sge = 2;
|
||||
}
|
||||
|
||||
/* For discovery session we re-use the login buffer */
|
||||
if (ic->ic_session_login_phase(ic) || ic->ic_session_type_discovery(ic)) {
|
||||
err = iser_post_recvl(iser_conn);
|
||||
if (err)
|
||||
goto send_control_error;
|
||||
}
|
||||
|
||||
err = iser_post_send(&iser_conn->ib_conn, mdesc, true);
|
||||
if (!err)
|
||||
return (0);
|
||||
|
||||
send_control_error:
|
||||
ISER_ERR("conn %p failed err %d", iser_conn, err);
|
||||
|
||||
return (err);
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_rcv_dto_completion - recv DTO completion
|
||||
*/
|
||||
void
|
||||
iser_rcv_completion(struct iser_rx_desc *rx_desc,
|
||||
unsigned long rx_xfer_len,
|
||||
struct ib_conn *ib_conn)
|
||||
{
|
||||
struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
|
||||
ib_conn);
|
||||
struct icl_conn *ic = &iser_conn->icl_conn;
|
||||
struct icl_pdu *response;
|
||||
struct iscsi_bhs *hdr;
|
||||
u64 rx_dma;
|
||||
int rx_buflen;
|
||||
int outstanding, count, err;
|
||||
|
||||
/* differentiate between login to all other PDUs */
|
||||
if ((char *)rx_desc == iser_conn->login_resp_buf) {
|
||||
rx_dma = iser_conn->login_resp_dma;
|
||||
rx_buflen = ISER_RX_LOGIN_SIZE;
|
||||
} else {
|
||||
rx_dma = rx_desc->dma_addr;
|
||||
rx_buflen = ISER_RX_PAYLOAD_SIZE;
|
||||
}
|
||||
|
||||
ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
|
||||
rx_buflen, DMA_FROM_DEVICE);
|
||||
|
||||
hdr = &rx_desc->iscsi_header;
|
||||
|
||||
response = iser_new_pdu(ic, M_NOWAIT);
|
||||
response->ip_bhs = hdr;
|
||||
response->ip_data_len = rx_xfer_len - ISER_HEADERS_LEN;
|
||||
|
||||
/*
|
||||
* In case we got data in the receive buffer, assign the ip_data_mbuf
|
||||
* to the rx_buffer - later we'll copy it to upper layer buffers
|
||||
*/
|
||||
if (response->ip_data_len)
|
||||
response->ip_data_mbuf = (struct mbuf *)(rx_desc->data);
|
||||
|
||||
ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
|
||||
rx_buflen, DMA_FROM_DEVICE);
|
||||
|
||||
/* decrementing conn->post_recv_buf_count only --after-- freeing the *
|
||||
* task eliminates the need to worry on tasks which are completed in *
|
||||
* parallel to the execution of iser_conn_term. So the code that waits *
|
||||
* for the posted rx bufs refcount to become zero handles everything */
|
||||
ib_conn->post_recv_buf_count--;
|
||||
|
||||
if (rx_dma == iser_conn->login_resp_dma)
|
||||
goto receive;
|
||||
|
||||
outstanding = ib_conn->post_recv_buf_count;
|
||||
if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
|
||||
count = min(iser_conn->qp_max_recv_dtos - outstanding,
|
||||
iser_conn->min_posted_rx);
|
||||
err = iser_post_recvm(iser_conn, count);
|
||||
if (err)
|
||||
ISER_ERR("posting %d rx bufs err %d", count, err);
|
||||
}
|
||||
|
||||
receive:
|
||||
(ic->ic_receive)(response);
|
||||
}
|
||||
|
||||
void
|
||||
iser_snd_completion(struct iser_tx_desc *tx_desc,
|
||||
struct ib_conn *ib_conn)
|
||||
{
|
||||
struct icl_iser_pdu *iser_pdu = container_of(tx_desc, struct icl_iser_pdu, desc);
|
||||
struct iser_conn *iser_conn = iser_pdu->iser_conn;
|
||||
|
||||
if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL)
|
||||
iser_pdu_free(&iser_conn->icl_conn, &iser_pdu->icl_pdu);
|
||||
}
|
348
sys/dev/iser/iser_memory.c
Normal file
348
sys/dev/iser/iser_memory.c
Normal file
@ -0,0 +1,348 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "icl_iser.h"
|
||||
|
||||
static struct fast_reg_descriptor *
|
||||
iser_reg_desc_get(struct ib_conn *ib_conn)
|
||||
{
|
||||
struct fast_reg_descriptor *desc;
|
||||
|
||||
mtx_lock(&ib_conn->lock);
|
||||
desc = list_first_entry(&ib_conn->fastreg.pool,
|
||||
struct fast_reg_descriptor, list);
|
||||
list_del(&desc->list);
|
||||
mtx_unlock(&ib_conn->lock);
|
||||
|
||||
return (desc);
|
||||
}
|
||||
|
||||
static void
|
||||
iser_reg_desc_put(struct ib_conn *ib_conn,
|
||||
struct fast_reg_descriptor *desc)
|
||||
{
|
||||
mtx_lock(&ib_conn->lock);
|
||||
list_add(&desc->list, &ib_conn->fastreg.pool);
|
||||
mtx_unlock(&ib_conn->lock);
|
||||
}
|
||||
|
||||
#define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0)
|
||||
|
||||
/**
|
||||
* iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
|
||||
* and returns the length of resulting physical address array (may be less than
|
||||
* the original due to possible compaction).
|
||||
*
|
||||
* we build a "page vec" under the assumption that the SG meets the RDMA
|
||||
* alignment requirements. Other then the first and last SG elements, all
|
||||
* the "internal" elements can be compacted into a list whose elements are
|
||||
* dma addresses of physical pages. The code supports also the weird case
|
||||
* where --few fragments of the same page-- are present in the SG as
|
||||
* consecutive elements. Also, it handles one entry SG.
|
||||
*/
|
||||
static int
|
||||
iser_sg_to_page_vec(struct iser_data_buf *data,
|
||||
struct ib_device *ibdev, u64 *pages,
|
||||
int *offset, int *data_size)
|
||||
{
|
||||
struct scatterlist *sg, *sgl = data->sgl;
|
||||
u64 start_addr, end_addr, page, chunk_start = 0;
|
||||
unsigned long total_sz = 0;
|
||||
unsigned int dma_len;
|
||||
int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
|
||||
|
||||
/* compute the offset of first element */
|
||||
*offset = (u64) sgl[0].offset & ~MASK_4K;
|
||||
|
||||
new_chunk = 1;
|
||||
cur_page = 0;
|
||||
for_each_sg(sgl, sg, data->dma_nents, i) {
|
||||
start_addr = ib_sg_dma_address(ibdev, sg);
|
||||
if (new_chunk)
|
||||
chunk_start = start_addr;
|
||||
dma_len = ib_sg_dma_len(ibdev, sg);
|
||||
end_addr = start_addr + dma_len;
|
||||
total_sz += dma_len;
|
||||
|
||||
/* collect page fragments until aligned or end of SG list */
|
||||
if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
|
||||
new_chunk = 0;
|
||||
continue;
|
||||
}
|
||||
new_chunk = 1;
|
||||
|
||||
/* address of the first page in the contiguous chunk;
|
||||
masking relevant for the very first SG entry,
|
||||
which might be unaligned */
|
||||
page = chunk_start & MASK_4K;
|
||||
do {
|
||||
pages[cur_page++] = page;
|
||||
page += SIZE_4K;
|
||||
} while (page < end_addr);
|
||||
}
|
||||
|
||||
*data_size = total_sz;
|
||||
|
||||
return (cur_page);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
|
||||
* for RDMA sub-list of a scatter-gather list of memory buffers, and returns
|
||||
* the number of entries which are aligned correctly. Supports the case where
|
||||
* consecutive SG elements are actually fragments of the same physcial page.
|
||||
*/
|
||||
static int
|
||||
iser_data_buf_aligned_len(struct iser_data_buf *data, struct ib_device *ibdev)
|
||||
{
|
||||
struct scatterlist *sg, *sgl, *next_sg = NULL;
|
||||
u64 start_addr, end_addr;
|
||||
int i, ret_len, start_check = 0;
|
||||
|
||||
if (data->dma_nents == 1)
|
||||
return (1);
|
||||
|
||||
sgl = data->sgl;
|
||||
start_addr = ib_sg_dma_address(ibdev, sgl);
|
||||
|
||||
for_each_sg(sgl, sg, data->dma_nents, i) {
|
||||
if (start_check && !IS_4K_ALIGNED(start_addr))
|
||||
break;
|
||||
|
||||
next_sg = sg_next(sg);
|
||||
if (!next_sg)
|
||||
break;
|
||||
|
||||
end_addr = start_addr + ib_sg_dma_len(ibdev, sg);
|
||||
start_addr = ib_sg_dma_address(ibdev, next_sg);
|
||||
|
||||
if (end_addr == start_addr) {
|
||||
start_check = 0;
|
||||
continue;
|
||||
} else
|
||||
start_check = 1;
|
||||
|
||||
if (!IS_4K_ALIGNED(end_addr))
|
||||
break;
|
||||
}
|
||||
ret_len = (next_sg) ? i : i+1;
|
||||
|
||||
return (ret_len);
|
||||
}
|
||||
|
||||
void
|
||||
iser_dma_unmap_task_data(struct icl_iser_pdu *iser_pdu,
|
||||
struct iser_data_buf *data,
|
||||
enum dma_data_direction dir)
|
||||
{
|
||||
struct ib_device *dev;
|
||||
|
||||
dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
|
||||
ib_dma_unmap_sg(dev, data->sgl, data->size, dir);
|
||||
}
|
||||
|
||||
static int
|
||||
iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem,
|
||||
struct iser_mem_reg *reg)
|
||||
{
|
||||
struct scatterlist *sg = mem->sgl;
|
||||
|
||||
reg->sge.lkey = device->mr->lkey;
|
||||
reg->rkey = device->mr->rkey;
|
||||
reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]);
|
||||
reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO: This should be a verb
|
||||
* iser_ib_inc_rkey - increments the key portion of the given rkey. Can be used
|
||||
* for calculating a new rkey for type 2 memory windows.
|
||||
* @rkey - the rkey to increment.
|
||||
*/
|
||||
static inline u32
|
||||
iser_ib_inc_rkey(u32 rkey)
|
||||
{
|
||||
const u32 mask = 0x000000ff;
|
||||
|
||||
return (((rkey + 1) & mask) | (rkey & ~mask));
|
||||
}
|
||||
|
||||
static void
|
||||
iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr)
|
||||
{
|
||||
u32 rkey;
|
||||
|
||||
memset(inv_wr, 0, sizeof(*inv_wr));
|
||||
inv_wr->opcode = IB_WR_LOCAL_INV;
|
||||
inv_wr->wr_id = ISER_FASTREG_LI_WRID;
|
||||
inv_wr->ex.invalidate_rkey = mr->rkey;
|
||||
|
||||
rkey = iser_ib_inc_rkey(mr->rkey);
|
||||
ib_update_fast_reg_key(mr, rkey);
|
||||
}
|
||||
|
||||
static int
|
||||
iser_fast_reg_mr(struct icl_iser_pdu *iser_pdu,
|
||||
struct iser_data_buf *mem,
|
||||
struct iser_reg_resources *rsc,
|
||||
struct iser_mem_reg *reg)
|
||||
{
|
||||
struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
struct ib_send_wr fastreg_wr, inv_wr;
|
||||
struct ib_send_wr *bad_wr, *wr = NULL;
|
||||
int ret, offset, size, plen;
|
||||
|
||||
/* if there a single dma entry, dma mr suffices */
|
||||
if (mem->dma_nents == 1)
|
||||
return iser_reg_dma(device, mem, reg);
|
||||
|
||||
/* rsc is not null */
|
||||
plen = iser_sg_to_page_vec(mem, device->ib_device,
|
||||
rsc->frpl->page_list,
|
||||
&offset, &size);
|
||||
if (plen * SIZE_4K < size) {
|
||||
ISER_ERR("fast reg page_list too short to hold this SG");
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
if (!rsc->mr_valid) {
|
||||
iser_inv_rkey(&inv_wr, rsc->mr);
|
||||
wr = &inv_wr;
|
||||
}
|
||||
|
||||
/* Prepare FASTREG WR */
|
||||
memset(&fastreg_wr, 0, sizeof(fastreg_wr));
|
||||
fastreg_wr.wr_id = ISER_FASTREG_LI_WRID;
|
||||
fastreg_wr.opcode = IB_WR_FAST_REG_MR;
|
||||
fastreg_wr.wr.fast_reg.iova_start = rsc->frpl->page_list[0] + offset;
|
||||
fastreg_wr.wr.fast_reg.page_list = rsc->frpl;
|
||||
fastreg_wr.wr.fast_reg.page_list_len = plen;
|
||||
fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
|
||||
fastreg_wr.wr.fast_reg.length = size;
|
||||
fastreg_wr.wr.fast_reg.rkey = rsc->mr->rkey;
|
||||
fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE |
|
||||
IB_ACCESS_REMOTE_WRITE |
|
||||
IB_ACCESS_REMOTE_READ);
|
||||
|
||||
if (!wr)
|
||||
wr = &fastreg_wr;
|
||||
else
|
||||
wr->next = &fastreg_wr;
|
||||
|
||||
ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
|
||||
if (ret) {
|
||||
ISER_ERR("fast registration failed, ret:%d", ret);
|
||||
return (ret);
|
||||
}
|
||||
rsc->mr_valid = 0;
|
||||
|
||||
reg->sge.lkey = rsc->mr->lkey;
|
||||
reg->rkey = rsc->mr->rkey;
|
||||
reg->sge.addr = rsc->frpl->page_list[0] + offset;
|
||||
reg->sge.length = size;
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_reg_rdma_mem - Registers memory intended for RDMA,
|
||||
* using Fast Registration WR (if possible) obtaining rkey and va
|
||||
*
|
||||
* returns 0 on success, errno code on failure
|
||||
*/
|
||||
int
|
||||
iser_reg_rdma_mem(struct icl_iser_pdu *iser_pdu,
|
||||
enum iser_data_dir cmd_dir)
|
||||
{
|
||||
struct ib_conn *ib_conn = &iser_pdu->iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
struct ib_device *ibdev = device->ib_device;
|
||||
struct iser_data_buf *mem = &iser_pdu->data[cmd_dir];
|
||||
struct iser_mem_reg *mem_reg = &iser_pdu->rdma_reg[cmd_dir];
|
||||
struct fast_reg_descriptor *desc = NULL;
|
||||
int err, aligned_len;
|
||||
|
||||
aligned_len = iser_data_buf_aligned_len(mem, ibdev);
|
||||
if (aligned_len != mem->dma_nents) {
|
||||
ISER_ERR("bounce buffer is not supported");
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (mem->dma_nents != 1) {
|
||||
desc = iser_reg_desc_get(ib_conn);
|
||||
mem_reg->mem_h = desc;
|
||||
}
|
||||
|
||||
err = iser_fast_reg_mr(iser_pdu, mem, desc ? &desc->rsc : NULL,
|
||||
mem_reg);
|
||||
if (err)
|
||||
goto err_reg;
|
||||
|
||||
return (0);
|
||||
|
||||
err_reg:
|
||||
if (desc)
|
||||
iser_reg_desc_put(ib_conn, desc);
|
||||
|
||||
return (err);
|
||||
}
|
||||
|
||||
void
|
||||
iser_unreg_rdma_mem(struct icl_iser_pdu *iser_pdu,
|
||||
enum iser_data_dir cmd_dir)
|
||||
{
|
||||
struct iser_mem_reg *reg = &iser_pdu->rdma_reg[cmd_dir];
|
||||
|
||||
if (!reg->mem_h)
|
||||
return;
|
||||
|
||||
iser_reg_desc_put(&iser_pdu->iser_conn->ib_conn,
|
||||
reg->mem_h);
|
||||
reg->mem_h = NULL;
|
||||
}
|
||||
|
||||
int
|
||||
iser_dma_map_task_data(struct icl_iser_pdu *iser_pdu,
|
||||
struct iser_data_buf *data,
|
||||
enum iser_data_dir iser_dir,
|
||||
enum dma_data_direction dma_dir)
|
||||
{
|
||||
struct ib_device *dev;
|
||||
|
||||
iser_pdu->dir[iser_dir] = 1;
|
||||
dev = iser_pdu->iser_conn->ib_conn.device->ib_device;
|
||||
|
||||
data->dma_nents = ib_dma_map_sg(dev, data->sgl, data->size, dma_dir);
|
||||
if (data->dma_nents == 0) {
|
||||
ISER_ERR("dma_map_sg failed");
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
965
sys/dev/iser/iser_verbs.c
Normal file
965
sys/dev/iser/iser_verbs.c
Normal file
@ -0,0 +1,965 @@
|
||||
/* $FreeBSD$ */
|
||||
/*-
|
||||
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "icl_iser.h"
|
||||
|
||||
static MALLOC_DEFINE(M_ISER_VERBS, "iser_verbs", "iser verbs backend");
|
||||
static int iser_cq_poll_limit = 512;
|
||||
|
||||
static void
|
||||
iser_cq_event_callback(struct ib_event *cause, void *context)
|
||||
{
|
||||
ISER_ERR("got cq event %d", cause->event);
|
||||
}
|
||||
|
||||
static void
|
||||
iser_qp_event_callback(struct ib_event *cause, void *context)
|
||||
{
|
||||
ISER_ERR("got qp event %d", cause->event);
|
||||
}
|
||||
|
||||
static void
|
||||
iser_event_handler(struct ib_event_handler *handler,
|
||||
struct ib_event *event)
|
||||
{
|
||||
ISER_ERR("async event %d on device %s port %d",
|
||||
event->event, event->device->name,
|
||||
event->element.port_num);
|
||||
}
|
||||
|
||||
/**
|
||||
* is_iser_tx_desc - Indicate if the completion wr_id
|
||||
* is a TX descriptor or not.
|
||||
* @iser_conn: iser connection
|
||||
* @wr_id: completion WR identifier
|
||||
*
|
||||
* Since we cannot rely on wc opcode in FLUSH errors
|
||||
* we must work around it by checking if the wr_id address
|
||||
* falls in the iser connection rx_descs buffer. If so
|
||||
* it is an RX descriptor, otherwize it is a TX.
|
||||
*/
|
||||
static inline bool
|
||||
is_iser_tx_desc(struct iser_conn *iser_conn, void *wr_id)
|
||||
{
|
||||
void *start = iser_conn->rx_descs;
|
||||
u64 len = iser_conn->num_rx_descs * sizeof(*iser_conn->rx_descs);
|
||||
void *end = (void *)((uintptr_t)start + (uintptr_t)len);
|
||||
|
||||
if (start) {
|
||||
if (wr_id >= start && wr_id < end)
|
||||
return false;
|
||||
} else {
|
||||
return ((uintptr_t)wr_id != (uintptr_t)iser_conn->login_resp_buf);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_handle_comp_error() - Handle error completion
|
||||
* @ib_conn: connection RDMA resources
|
||||
* @wc: work completion
|
||||
*
|
||||
* Notes: Update post_recv_buf_count in case of recv error completion.
|
||||
* For non-FLUSH error completion we should also notify iscsi layer that
|
||||
* connection is failed (in case we passed bind stage).
|
||||
*/
|
||||
static void
|
||||
iser_handle_comp_error(struct ib_conn *ib_conn,
|
||||
struct ib_wc *wc)
|
||||
{
|
||||
void *wr_id = (void *)(uintptr_t)wc->wr_id;
|
||||
struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
|
||||
ib_conn);
|
||||
|
||||
if (is_iser_tx_desc(iser_conn, wr_id)) {
|
||||
ISER_DBG("conn %p got send comp error", iser_conn);
|
||||
} else {
|
||||
ISER_DBG("conn %p got recv comp error", iser_conn);
|
||||
ib_conn->post_recv_buf_count--;
|
||||
}
|
||||
if (wc->status != IB_WC_WR_FLUSH_ERR)
|
||||
iser_conn->icl_conn.ic_error(&iser_conn->icl_conn);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_handle_wc - handle a single work completion
|
||||
* @wc: work completion
|
||||
*
|
||||
* Soft-IRQ context, work completion can be either
|
||||
* SEND or RECV, and can turn out successful or
|
||||
* with error (or flush error).
|
||||
*/
|
||||
static void iser_handle_wc(struct ib_wc *wc)
|
||||
{
|
||||
struct ib_conn *ib_conn;
|
||||
struct iser_tx_desc *tx_desc;
|
||||
struct iser_rx_desc *rx_desc;
|
||||
|
||||
ib_conn = wc->qp->qp_context;
|
||||
if (likely(wc->status == IB_WC_SUCCESS)) {
|
||||
if (wc->opcode == IB_WC_RECV) {
|
||||
rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id;
|
||||
iser_rcv_completion(rx_desc, wc->byte_len,
|
||||
ib_conn);
|
||||
} else
|
||||
if (wc->opcode == IB_WC_SEND) {
|
||||
tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id;
|
||||
iser_snd_completion(tx_desc, ib_conn);
|
||||
} else {
|
||||
ISER_ERR("Unknown wc opcode %d", wc->opcode);
|
||||
}
|
||||
} else {
|
||||
struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
|
||||
ib_conn);
|
||||
if (wc->status != IB_WC_WR_FLUSH_ERR) {
|
||||
ISER_ERR("conn %p wr id %lx status %d vend_err %x",
|
||||
iser_conn, wc->wr_id, wc->status, wc->vendor_err);
|
||||
} else {
|
||||
ISER_DBG("flush error: conn %p wr id %lx", iser_conn, wc->wr_id);
|
||||
}
|
||||
|
||||
if (wc->wr_id == ISER_BEACON_WRID) {
|
||||
/* all flush errors were consumed */
|
||||
mtx_lock(&ib_conn->beacon.flush_lock);
|
||||
ISER_DBG("conn %p got ISER_BEACON_WRID", iser_conn);
|
||||
cv_signal(&ib_conn->beacon.flush_cv);
|
||||
mtx_unlock(&ib_conn->beacon.flush_lock);
|
||||
} else {
|
||||
iser_handle_comp_error(ib_conn, wc);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
iser_cq_tasklet_fn(void *data, int pending)
|
||||
{
|
||||
struct iser_comp *comp = (struct iser_comp *)data;
|
||||
struct ib_cq *cq = comp->cq;
|
||||
struct ib_wc *const wcs = comp->wcs;
|
||||
int completed = 0;
|
||||
int i;
|
||||
int n;
|
||||
|
||||
while ((n = ib_poll_cq(cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) {
|
||||
for (i = 0; i < n; i++)
|
||||
iser_handle_wc(&wcs[i]);
|
||||
|
||||
completed += n;
|
||||
if (completed >= iser_cq_poll_limit)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* It is assumed here that arming CQ only once its empty
|
||||
* would not cause interrupts to be missed.
|
||||
*/
|
||||
ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
|
||||
}
|
||||
|
||||
static void
|
||||
iser_cq_callback(struct ib_cq *cq, void *cq_context)
|
||||
{
|
||||
struct iser_comp *comp = cq_context;
|
||||
|
||||
taskqueue_enqueue_fast(comp->tq, &comp->task);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_create_device_ib_res - creates Protection Domain (PD), Completion
|
||||
* Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
|
||||
* the adapator.
|
||||
*
|
||||
* returns 0 on success, -1 on failure
|
||||
*/
|
||||
static int
|
||||
iser_create_device_ib_res(struct iser_device *device)
|
||||
{
|
||||
struct ib_device_attr *dev_attr = &device->dev_attr;
|
||||
int ret, i, max_cqe;
|
||||
|
||||
ret = ib_query_device(device->ib_device, dev_attr);
|
||||
if (ret) {
|
||||
ISER_ERR("Query device failed for %s", device->ib_device->name);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
if (!(dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) {
|
||||
ISER_ERR("device %s doesn't support Fastreg, "
|
||||
"can't register memory", device->ib_device->name);
|
||||
return (1);
|
||||
}
|
||||
|
||||
device->comps_used = min(mp_ncpus, device->ib_device->num_comp_vectors);
|
||||
|
||||
device->comps = malloc(device->comps_used * sizeof(*device->comps),
|
||||
M_ISER_VERBS, M_WAITOK | M_ZERO);
|
||||
if (!device->comps)
|
||||
goto comps_err;
|
||||
|
||||
max_cqe = min(ISER_MAX_CQ_LEN, dev_attr->max_cqe);
|
||||
|
||||
ISER_DBG("using %d CQs, device %s supports %d vectors max_cqe %d",
|
||||
device->comps_used, device->ib_device->name,
|
||||
device->ib_device->num_comp_vectors, max_cqe);
|
||||
|
||||
device->pd = ib_alloc_pd(device->ib_device);
|
||||
if (IS_ERR(device->pd))
|
||||
goto pd_err;
|
||||
|
||||
for (i = 0; i < device->comps_used; i++) {
|
||||
struct iser_comp *comp = &device->comps[i];
|
||||
|
||||
comp->device = device;
|
||||
comp->cq = ib_create_cq(device->ib_device,
|
||||
iser_cq_callback,
|
||||
iser_cq_event_callback,
|
||||
(void *)comp,
|
||||
max_cqe, i);
|
||||
if (IS_ERR(comp->cq)) {
|
||||
comp->cq = NULL;
|
||||
goto cq_err;
|
||||
}
|
||||
|
||||
if (ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP))
|
||||
goto cq_err;
|
||||
|
||||
TASK_INIT(&comp->task, 0, iser_cq_tasklet_fn, comp);
|
||||
comp->tq = taskqueue_create_fast("iser_taskq", M_NOWAIT,
|
||||
taskqueue_thread_enqueue, &comp->tq);
|
||||
if (!comp->tq)
|
||||
goto tq_err;
|
||||
taskqueue_start_threads(&comp->tq, 1, PI_NET, "iser taskq");
|
||||
}
|
||||
|
||||
device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE |
|
||||
IB_ACCESS_REMOTE_WRITE |
|
||||
IB_ACCESS_REMOTE_READ);
|
||||
if (IS_ERR(device->mr))
|
||||
goto tq_err;
|
||||
|
||||
INIT_IB_EVENT_HANDLER(&device->event_handler, device->ib_device,
|
||||
iser_event_handler);
|
||||
if (ib_register_event_handler(&device->event_handler))
|
||||
goto handler_err;
|
||||
|
||||
return (0);
|
||||
|
||||
handler_err:
|
||||
ib_dereg_mr(device->mr);
|
||||
tq_err:
|
||||
for (i = 0; i < device->comps_used; i++) {
|
||||
struct iser_comp *comp = &device->comps[i];
|
||||
if (comp->tq)
|
||||
taskqueue_free(comp->tq);
|
||||
}
|
||||
cq_err:
|
||||
for (i = 0; i < device->comps_used; i++) {
|
||||
struct iser_comp *comp = &device->comps[i];
|
||||
if (comp->cq)
|
||||
ib_destroy_cq(comp->cq);
|
||||
}
|
||||
ib_dealloc_pd(device->pd);
|
||||
pd_err:
|
||||
free(device->comps, M_ISER_VERBS);
|
||||
comps_err:
|
||||
ISER_ERR("failed to allocate an IB resource");
|
||||
return (1);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
|
||||
* CQ and PD created with the device associated with the adapator.
|
||||
*/
|
||||
static void
|
||||
iser_free_device_ib_res(struct iser_device *device)
|
||||
{
|
||||
int i;
|
||||
|
||||
for (i = 0; i < device->comps_used; i++) {
|
||||
struct iser_comp *comp = &device->comps[i];
|
||||
|
||||
taskqueue_free(comp->tq);
|
||||
ib_destroy_cq(comp->cq);
|
||||
comp->cq = NULL;
|
||||
}
|
||||
|
||||
(void)ib_unregister_event_handler(&device->event_handler);
|
||||
(void)ib_dereg_mr(device->mr);
|
||||
(void)ib_dealloc_pd(device->pd);
|
||||
|
||||
free(device->comps, M_ISER_VERBS);
|
||||
device->comps = NULL;
|
||||
|
||||
device->mr = NULL;
|
||||
device->pd = NULL;
|
||||
}
|
||||
|
||||
static int
|
||||
iser_alloc_reg_res(struct ib_device *ib_device,
|
||||
struct ib_pd *pd,
|
||||
struct iser_reg_resources *res)
|
||||
{
|
||||
int ret;
|
||||
|
||||
res->frpl = ib_alloc_fast_reg_page_list(ib_device,
|
||||
ISCSI_ISER_SG_TABLESIZE + 1);
|
||||
if (IS_ERR(res->frpl)) {
|
||||
ret = -PTR_ERR(res->frpl);
|
||||
ISER_ERR("Failed to allocate fast reg page list err=%d", ret);
|
||||
return (ret);
|
||||
}
|
||||
|
||||
res->mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE + 1);
|
||||
if (IS_ERR(res->mr)) {
|
||||
ret = -PTR_ERR(res->mr);
|
||||
ISER_ERR("Failed to allocate fast reg mr err=%d", ret);
|
||||
goto fast_reg_mr_failure;
|
||||
}
|
||||
res->mr_valid = 1;
|
||||
|
||||
return (0);
|
||||
|
||||
fast_reg_mr_failure:
|
||||
ib_free_fast_reg_page_list(res->frpl);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
static void
|
||||
iser_free_reg_res(struct iser_reg_resources *rsc)
|
||||
{
|
||||
ib_dereg_mr(rsc->mr);
|
||||
ib_free_fast_reg_page_list(rsc->frpl);
|
||||
}
|
||||
|
||||
static struct fast_reg_descriptor *
|
||||
iser_create_fastreg_desc(struct ib_device *ib_device, struct ib_pd *pd)
|
||||
{
|
||||
struct fast_reg_descriptor *desc;
|
||||
int ret;
|
||||
|
||||
desc = malloc(sizeof(*desc), M_ISER_VERBS, M_WAITOK | M_ZERO);
|
||||
if (!desc) {
|
||||
ISER_ERR("Failed to allocate a new fastreg descriptor");
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
ret = iser_alloc_reg_res(ib_device, pd, &desc->rsc);
|
||||
if (ret) {
|
||||
ISER_ERR("failed to allocate reg_resources");
|
||||
goto err;
|
||||
}
|
||||
|
||||
return (desc);
|
||||
err:
|
||||
free(desc, M_ISER_VERBS);
|
||||
return (NULL);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_create_fmr_pool - Creates FMR pool and page_vector
|
||||
*
|
||||
* returns 0 on success, or errno code on failure
|
||||
*/
|
||||
int
|
||||
iser_create_fastreg_pool(struct ib_conn *ib_conn, unsigned cmds_max)
|
||||
{
|
||||
struct iser_device *device = ib_conn->device;
|
||||
struct fast_reg_descriptor *desc;
|
||||
int i;
|
||||
|
||||
INIT_LIST_HEAD(&ib_conn->fastreg.pool);
|
||||
ib_conn->fastreg.pool_size = 0;
|
||||
for (i = 0; i < cmds_max; i++) {
|
||||
desc = iser_create_fastreg_desc(device->ib_device, device->pd);
|
||||
if (!desc) {
|
||||
ISER_ERR("Failed to create fastreg descriptor");
|
||||
goto err;
|
||||
}
|
||||
|
||||
list_add_tail(&desc->list, &ib_conn->fastreg.pool);
|
||||
ib_conn->fastreg.pool_size++;
|
||||
}
|
||||
|
||||
return (0);
|
||||
|
||||
err:
|
||||
iser_free_fastreg_pool(ib_conn);
|
||||
return (ENOMEM);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_free_fmr_pool - releases the FMR pool and page vec
|
||||
*/
|
||||
void
|
||||
iser_free_fastreg_pool(struct ib_conn *ib_conn)
|
||||
{
|
||||
struct fast_reg_descriptor *desc, *tmp;
|
||||
int i = 0;
|
||||
|
||||
if (list_empty(&ib_conn->fastreg.pool))
|
||||
return;
|
||||
|
||||
ISER_DBG("freeing conn %p fr pool", ib_conn);
|
||||
|
||||
list_for_each_entry_safe(desc, tmp, &ib_conn->fastreg.pool, list) {
|
||||
list_del(&desc->list);
|
||||
iser_free_reg_res(&desc->rsc);
|
||||
free(desc, M_ISER_VERBS);
|
||||
++i;
|
||||
}
|
||||
|
||||
if (i < ib_conn->fastreg.pool_size)
|
||||
ISER_WARN("pool still has %d regions registered",
|
||||
ib_conn->fastreg.pool_size - i);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_create_ib_conn_res - Queue-Pair (QP)
|
||||
*
|
||||
* returns 0 on success, 1 on failure
|
||||
*/
|
||||
static int
|
||||
iser_create_ib_conn_res(struct ib_conn *ib_conn)
|
||||
{
|
||||
struct iser_conn *iser_conn;
|
||||
struct iser_device *device;
|
||||
struct ib_device_attr *dev_attr;
|
||||
struct ib_qp_init_attr init_attr;
|
||||
int index, min_index = 0;
|
||||
int ret = -ENOMEM;
|
||||
|
||||
iser_conn = container_of(ib_conn, struct iser_conn, ib_conn);
|
||||
device = ib_conn->device;
|
||||
dev_attr = &device->dev_attr;
|
||||
|
||||
mtx_lock(&ig.connlist_mutex);
|
||||
/* select the CQ with the minimal number of usages */
|
||||
for (index = 0; index < device->comps_used; index++) {
|
||||
if (device->comps[index].active_qps <
|
||||
device->comps[min_index].active_qps)
|
||||
min_index = index;
|
||||
}
|
||||
ib_conn->comp = &device->comps[min_index];
|
||||
ib_conn->comp->active_qps++;
|
||||
mtx_unlock(&ig.connlist_mutex);
|
||||
ISER_INFO("cq index %d used for ib_conn %p", min_index, ib_conn);
|
||||
|
||||
memset(&init_attr, 0, sizeof init_attr);
|
||||
init_attr.event_handler = iser_qp_event_callback;
|
||||
init_attr.qp_context = (void *)ib_conn;
|
||||
init_attr.send_cq = ib_conn->comp->cq;
|
||||
init_attr.recv_cq = ib_conn->comp->cq;
|
||||
init_attr.cap.max_recv_wr = ISER_QP_MAX_RECV_DTOS;
|
||||
init_attr.cap.max_send_sge = 2;
|
||||
init_attr.cap.max_recv_sge = 1;
|
||||
init_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
|
||||
init_attr.qp_type = IB_QPT_RC;
|
||||
|
||||
if (dev_attr->max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
|
||||
init_attr.cap.max_send_wr = ISER_QP_MAX_REQ_DTOS;
|
||||
iser_conn->max_cmds =
|
||||
ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
|
||||
} else {
|
||||
init_attr.cap.max_send_wr = dev_attr->max_qp_wr;
|
||||
iser_conn->max_cmds =
|
||||
ISER_GET_MAX_XMIT_CMDS(dev_attr->max_qp_wr);
|
||||
}
|
||||
ISER_DBG("device %s supports max_send_wr %d",
|
||||
device->ib_device->name, dev_attr->max_qp_wr);
|
||||
|
||||
ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
|
||||
if (ret)
|
||||
goto out_err;
|
||||
|
||||
ib_conn->qp = ib_conn->cma_id->qp;
|
||||
ISER_DBG("setting conn %p cma_id %p qp %p",
|
||||
ib_conn, ib_conn->cma_id,
|
||||
ib_conn->cma_id->qp);
|
||||
|
||||
return (ret);
|
||||
|
||||
out_err:
|
||||
mtx_lock(&ig.connlist_mutex);
|
||||
ib_conn->comp->active_qps--;
|
||||
mtx_unlock(&ig.connlist_mutex);
|
||||
ISER_ERR("unable to alloc mem or create resource, err %d", ret);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
/**
|
||||
* based on the resolved device node GUID see if there already allocated
|
||||
* device for this device. If there's no such, create one.
|
||||
*/
|
||||
static struct iser_device *
|
||||
iser_device_find_by_ib_device(struct rdma_cm_id *cma_id)
|
||||
{
|
||||
struct iser_device *device;
|
||||
|
||||
sx_xlock(&ig.device_list_mutex);
|
||||
|
||||
list_for_each_entry(device, &ig.device_list, ig_list)
|
||||
/* find if there's a match using the node GUID */
|
||||
if (device->ib_device->node_guid == cma_id->device->node_guid)
|
||||
goto inc_refcnt;
|
||||
|
||||
device = malloc(sizeof *device, M_ISER_VERBS, M_WAITOK | M_ZERO);
|
||||
if (device == NULL)
|
||||
goto out;
|
||||
|
||||
/* assign this device to the device */
|
||||
device->ib_device = cma_id->device;
|
||||
/* init the device and link it into ig device list */
|
||||
if (iser_create_device_ib_res(device)) {
|
||||
free(device, M_ISER_VERBS);
|
||||
device = NULL;
|
||||
goto out;
|
||||
}
|
||||
list_add(&device->ig_list, &ig.device_list);
|
||||
|
||||
inc_refcnt:
|
||||
device->refcount++;
|
||||
ISER_INFO("device %p refcount %d", device, device->refcount);
|
||||
out:
|
||||
sx_xunlock(&ig.device_list_mutex);
|
||||
return (device);
|
||||
}
|
||||
|
||||
/* if there's no demand for this device, release it */
|
||||
static void
|
||||
iser_device_try_release(struct iser_device *device)
|
||||
{
|
||||
sx_xlock(&ig.device_list_mutex);
|
||||
device->refcount--;
|
||||
ISER_INFO("device %p refcount %d", device, device->refcount);
|
||||
if (!device->refcount) {
|
||||
iser_free_device_ib_res(device);
|
||||
list_del(&device->ig_list);
|
||||
free(device, M_ISER_VERBS);
|
||||
device = NULL;
|
||||
}
|
||||
sx_xunlock(&ig.device_list_mutex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
|
||||
enum iser_conn_state comp,
|
||||
enum iser_conn_state exch)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = (iser_conn->state == comp);
|
||||
if (ret)
|
||||
iser_conn->state = exch;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_free_ib_conn_res - release IB related resources
|
||||
* @iser_conn: iser connection struct
|
||||
* @destroy: indicator if we need to try to release the
|
||||
* iser device and memory regoins pool (only iscsi
|
||||
* shutdown and DEVICE_REMOVAL will use this).
|
||||
*
|
||||
* This routine is called with the iser state mutex held
|
||||
* so the cm_id removal is out of here. It is Safe to
|
||||
* be invoked multiple times.
|
||||
*/
|
||||
void
|
||||
iser_free_ib_conn_res(struct iser_conn *iser_conn,
|
||||
bool destroy)
|
||||
{
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
|
||||
ISER_INFO("freeing conn %p cma_id %p qp %p",
|
||||
iser_conn, ib_conn->cma_id, ib_conn->qp);
|
||||
|
||||
if (ib_conn->qp != NULL) {
|
||||
mtx_lock(&ig.connlist_mutex);
|
||||
ib_conn->comp->active_qps--;
|
||||
mtx_unlock(&ig.connlist_mutex);
|
||||
rdma_destroy_qp(ib_conn->cma_id);
|
||||
ib_conn->qp = NULL;
|
||||
}
|
||||
|
||||
if (destroy) {
|
||||
if (iser_conn->login_buf)
|
||||
iser_free_login_buf(iser_conn);
|
||||
|
||||
if (iser_conn->rx_descs)
|
||||
iser_free_rx_descriptors(iser_conn);
|
||||
|
||||
if (device != NULL) {
|
||||
iser_device_try_release(device);
|
||||
ib_conn->device = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* triggers start of the disconnect procedures and wait for them to be done
|
||||
* Called with state mutex held
|
||||
*/
|
||||
int
|
||||
iser_conn_terminate(struct iser_conn *iser_conn)
|
||||
{
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct ib_send_wr *bad_send_wr;
|
||||
struct ib_recv_wr *bad_recv_wr;
|
||||
int err = 0;
|
||||
|
||||
/* terminate the iser conn only if the conn state is UP */
|
||||
if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
|
||||
ISER_CONN_TERMINATING))
|
||||
return (0);
|
||||
|
||||
ISER_INFO("iser_conn %p state %d\n", iser_conn, iser_conn->state);
|
||||
|
||||
if (ib_conn->qp == NULL) {
|
||||
/* HOW can this be??? */
|
||||
ISER_WARN("qp wasn't created");
|
||||
return (1);
|
||||
}
|
||||
|
||||
/*
|
||||
* Todo: This is a temporary workaround.
|
||||
* We serialize the connection closure using global lock in order to
|
||||
* receive all posted beacons completions.
|
||||
* Without Serialization, in case we open many connections (QPs) on
|
||||
* the same CQ, we might miss beacons because of missing interrupts.
|
||||
*/
|
||||
sx_xlock(&ig.close_conns_mutex);
|
||||
|
||||
/*
|
||||
* In case we didn't already clean up the cma_id (peer initiated
|
||||
* a disconnection), we need to Cause the CMA to change the QP
|
||||
* state to ERROR.
|
||||
*/
|
||||
if (ib_conn->cma_id) {
|
||||
err = rdma_disconnect(ib_conn->cma_id);
|
||||
if (err)
|
||||
ISER_ERR("Failed to disconnect, conn: 0x%p err %d",
|
||||
iser_conn, err);
|
||||
|
||||
mtx_lock(&ib_conn->beacon.flush_lock);
|
||||
memset(&ib_conn->beacon.send, 0, sizeof(struct ib_send_wr));
|
||||
ib_conn->beacon.send.wr_id = ISER_BEACON_WRID;
|
||||
ib_conn->beacon.send.opcode = IB_WR_SEND;
|
||||
/* post an indication that all send flush errors were consumed */
|
||||
err = ib_post_send(ib_conn->qp, &ib_conn->beacon.send, &bad_send_wr);
|
||||
if (err) {
|
||||
ISER_ERR("conn %p failed to post send_beacon", ib_conn);
|
||||
mtx_unlock(&ib_conn->beacon.flush_lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
ISER_DBG("before send cv_wait: %p", iser_conn);
|
||||
cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock);
|
||||
ISER_DBG("after send cv_wait: %p", iser_conn);
|
||||
|
||||
memset(&ib_conn->beacon.recv, 0, sizeof(struct ib_recv_wr));
|
||||
ib_conn->beacon.recv.wr_id = ISER_BEACON_WRID;
|
||||
/* post an indication that all recv flush errors were consumed */
|
||||
err = ib_post_recv(ib_conn->qp, &ib_conn->beacon.recv, &bad_recv_wr);
|
||||
if (err) {
|
||||
ISER_ERR("conn %p failed to post recv_beacon", ib_conn);
|
||||
mtx_unlock(&ib_conn->beacon.flush_lock);
|
||||
goto out;
|
||||
}
|
||||
|
||||
ISER_DBG("before recv cv_wait: %p", iser_conn);
|
||||
cv_wait(&ib_conn->beacon.flush_cv, &ib_conn->beacon.flush_lock);
|
||||
mtx_unlock(&ib_conn->beacon.flush_lock);
|
||||
ISER_DBG("after recv cv_wait: %p", iser_conn);
|
||||
}
|
||||
out:
|
||||
sx_xunlock(&ig.close_conns_mutex);
|
||||
return (1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static void
|
||||
iser_connect_error(struct rdma_cm_id *cma_id)
|
||||
{
|
||||
struct iser_conn *iser_conn;
|
||||
|
||||
iser_conn = cma_id->context;
|
||||
|
||||
ISER_ERR("conn %p", iser_conn);
|
||||
|
||||
iser_conn->state = ISER_CONN_TERMINATING;
|
||||
|
||||
cv_signal(&iser_conn->up_cv);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static void
|
||||
iser_addr_handler(struct rdma_cm_id *cma_id)
|
||||
{
|
||||
struct iser_device *device;
|
||||
struct iser_conn *iser_conn;
|
||||
struct ib_conn *ib_conn;
|
||||
int ret;
|
||||
|
||||
iser_conn = cma_id->context;
|
||||
|
||||
ib_conn = &iser_conn->ib_conn;
|
||||
device = iser_device_find_by_ib_device(cma_id);
|
||||
if (!device) {
|
||||
ISER_ERR("conn %p device lookup/creation failed",
|
||||
iser_conn);
|
||||
iser_connect_error(cma_id);
|
||||
return;
|
||||
}
|
||||
|
||||
ib_conn->device = device;
|
||||
|
||||
ret = rdma_resolve_route(cma_id, 1000);
|
||||
if (ret) {
|
||||
ISER_ERR("conn %p resolve route failed: %d", iser_conn, ret);
|
||||
iser_connect_error(cma_id);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static void
|
||||
iser_route_handler(struct rdma_cm_id *cma_id)
|
||||
{
|
||||
struct rdma_conn_param conn_param;
|
||||
int ret;
|
||||
struct iser_cm_hdr req_hdr;
|
||||
struct iser_conn *iser_conn = cma_id->context;
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct iser_device *device = ib_conn->device;
|
||||
|
||||
ret = iser_create_ib_conn_res(ib_conn);
|
||||
if (ret)
|
||||
goto failure;
|
||||
|
||||
memset(&conn_param, 0, sizeof conn_param);
|
||||
conn_param.responder_resources = device->dev_attr.max_qp_rd_atom;
|
||||
conn_param.retry_count = 7;
|
||||
conn_param.rnr_retry_count = 6;
|
||||
/*
|
||||
* Initiaotr depth should not be set, but in order to compat
|
||||
* with old targets, we keep this value set.
|
||||
*/
|
||||
conn_param.initiator_depth = 1;
|
||||
|
||||
memset(&req_hdr, 0, sizeof(req_hdr));
|
||||
req_hdr.flags = (ISER_ZBVA_NOT_SUPPORTED |
|
||||
ISER_SEND_W_INV_NOT_SUPPORTED);
|
||||
conn_param.private_data = (void *)&req_hdr;
|
||||
conn_param.private_data_len = sizeof(struct iser_cm_hdr);
|
||||
|
||||
ret = rdma_connect(cma_id, &conn_param);
|
||||
if (ret) {
|
||||
ISER_ERR("conn %p failure connecting: %d", iser_conn, ret);
|
||||
goto failure;
|
||||
}
|
||||
|
||||
return;
|
||||
failure:
|
||||
iser_connect_error(cma_id);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static void
|
||||
iser_connected_handler(struct rdma_cm_id *cma_id)
|
||||
{
|
||||
struct iser_conn *iser_conn;
|
||||
struct ib_qp_attr attr;
|
||||
struct ib_qp_init_attr init_attr;
|
||||
|
||||
iser_conn = cma_id->context;
|
||||
|
||||
(void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);
|
||||
|
||||
ISER_INFO("remote qpn:%x my qpn:%x",
|
||||
attr.dest_qp_num, cma_id->qp->qp_num);
|
||||
|
||||
iser_conn->state = ISER_CONN_UP;
|
||||
|
||||
cv_signal(&iser_conn->up_cv);
|
||||
}
|
||||
|
||||
/**
|
||||
* Called with state mutex held
|
||||
**/
|
||||
static void
|
||||
iser_cleanup_handler(struct rdma_cm_id *cma_id, bool destroy)
|
||||
{
|
||||
struct iser_conn *iser_conn = cma_id->context;
|
||||
|
||||
if (iser_conn_terminate(iser_conn))
|
||||
iser_conn->icl_conn.ic_error(&iser_conn->icl_conn);
|
||||
|
||||
}
|
||||
|
||||
int
|
||||
iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
|
||||
{
|
||||
struct iser_conn *iser_conn;
|
||||
int ret = 0;
|
||||
|
||||
iser_conn = cma_id->context;
|
||||
ISER_INFO("event %d status %d conn %p id %p",
|
||||
event->event, event->status, cma_id->context, cma_id);
|
||||
|
||||
sx_xlock(&iser_conn->state_mutex);
|
||||
switch (event->event) {
|
||||
case RDMA_CM_EVENT_ADDR_RESOLVED:
|
||||
iser_addr_handler(cma_id);
|
||||
break;
|
||||
case RDMA_CM_EVENT_ROUTE_RESOLVED:
|
||||
iser_route_handler(cma_id);
|
||||
break;
|
||||
case RDMA_CM_EVENT_ESTABLISHED:
|
||||
iser_connected_handler(cma_id);
|
||||
break;
|
||||
case RDMA_CM_EVENT_ADDR_ERROR:
|
||||
case RDMA_CM_EVENT_ROUTE_ERROR:
|
||||
case RDMA_CM_EVENT_CONNECT_ERROR:
|
||||
case RDMA_CM_EVENT_UNREACHABLE:
|
||||
case RDMA_CM_EVENT_REJECTED:
|
||||
iser_connect_error(cma_id);
|
||||
break;
|
||||
case RDMA_CM_EVENT_DISCONNECTED:
|
||||
case RDMA_CM_EVENT_ADDR_CHANGE:
|
||||
case RDMA_CM_EVENT_TIMEWAIT_EXIT:
|
||||
iser_cleanup_handler(cma_id, false);
|
||||
break;
|
||||
default:
|
||||
ISER_ERR("Unexpected RDMA CM event (%d)", event->event);
|
||||
break;
|
||||
}
|
||||
sx_xunlock(&iser_conn->state_mutex);
|
||||
|
||||
return (ret);
|
||||
}
|
||||
|
||||
int
|
||||
iser_post_recvl(struct iser_conn *iser_conn)
|
||||
{
|
||||
struct ib_recv_wr rx_wr, *rx_wr_failed;
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
struct ib_sge sge;
|
||||
int ib_ret;
|
||||
|
||||
sge.addr = iser_conn->login_resp_dma;
|
||||
sge.length = ISER_RX_LOGIN_SIZE;
|
||||
sge.lkey = ib_conn->device->mr->lkey;
|
||||
|
||||
rx_wr.wr_id = (uintptr_t)iser_conn->login_resp_buf;
|
||||
rx_wr.sg_list = &sge;
|
||||
rx_wr.num_sge = 1;
|
||||
rx_wr.next = NULL;
|
||||
|
||||
ib_conn->post_recv_buf_count++;
|
||||
ib_ret = ib_post_recv(ib_conn->qp, &rx_wr, &rx_wr_failed);
|
||||
if (ib_ret) {
|
||||
ISER_ERR("ib_post_recv failed ret=%d", ib_ret);
|
||||
ib_conn->post_recv_buf_count--;
|
||||
}
|
||||
|
||||
return (ib_ret);
|
||||
}
|
||||
|
||||
int
|
||||
iser_post_recvm(struct iser_conn *iser_conn, int count)
|
||||
{
|
||||
struct ib_recv_wr *rx_wr, *rx_wr_failed;
|
||||
int i, ib_ret;
|
||||
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
||||
unsigned int my_rx_head = iser_conn->rx_desc_head;
|
||||
struct iser_rx_desc *rx_desc;
|
||||
|
||||
for (rx_wr = ib_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
|
||||
rx_desc = &iser_conn->rx_descs[my_rx_head];
|
||||
rx_wr->wr_id = (uintptr_t)rx_desc;
|
||||
rx_wr->sg_list = &rx_desc->rx_sg;
|
||||
rx_wr->num_sge = 1;
|
||||
rx_wr->next = rx_wr + 1;
|
||||
my_rx_head = (my_rx_head + 1) % iser_conn->qp_max_recv_dtos;
|
||||
}
|
||||
|
||||
rx_wr--;
|
||||
rx_wr->next = NULL; /* mark end of work requests list */
|
||||
|
||||
ib_conn->post_recv_buf_count += count;
|
||||
ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, &rx_wr_failed);
|
||||
if (ib_ret) {
|
||||
ISER_ERR("ib_post_recv failed ret=%d", ib_ret);
|
||||
ib_conn->post_recv_buf_count -= count;
|
||||
} else
|
||||
iser_conn->rx_desc_head = my_rx_head;
|
||||
|
||||
return (ib_ret);
|
||||
}
|
||||
|
||||
/**
|
||||
* iser_start_send - Initiate a Send DTO operation
|
||||
*
|
||||
* returns 0 on success, -1 on failure
|
||||
*/
|
||||
int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
|
||||
bool signal)
|
||||
{
|
||||
int ib_ret;
|
||||
struct ib_send_wr send_wr, *send_wr_failed;
|
||||
|
||||
ib_dma_sync_single_for_device(ib_conn->device->ib_device,
|
||||
tx_desc->dma_addr, ISER_HEADERS_LEN,
|
||||
DMA_TO_DEVICE);
|
||||
|
||||
send_wr.next = NULL;
|
||||
send_wr.wr_id = (uintptr_t)tx_desc;
|
||||
send_wr.sg_list = tx_desc->tx_sg;
|
||||
send_wr.num_sge = tx_desc->num_sge;
|
||||
send_wr.opcode = IB_WR_SEND;
|
||||
send_wr.send_flags = signal ? IB_SEND_SIGNALED : 0;
|
||||
|
||||
ib_ret = ib_post_send(ib_conn->qp, &send_wr, &send_wr_failed);
|
||||
if (ib_ret)
|
||||
ISER_ERR("ib_post_send failed, ret:%d", ib_ret);
|
||||
|
||||
return (ib_ret);
|
||||
}
|
32
sys/modules/iser/Makefile
Normal file
32
sys/modules/iser/Makefile
Normal file
@ -0,0 +1,32 @@
|
||||
# $FreeBSD$
|
||||
|
||||
.PATH: ${.CURDIR}/../../dev/iser/
|
||||
|
||||
.include <bsd.own.mk>
|
||||
|
||||
KMOD= iser
|
||||
|
||||
SRCS= icl_iser.c
|
||||
SRCS+= iser_initiator.c
|
||||
SRCS+= iser_memory.c
|
||||
SRCS+= iser_verbs.c
|
||||
SRCS+= vnode_if.h
|
||||
SRCS+= opt_inet.h
|
||||
SRCS+= opt_inet6.h
|
||||
SRCS+= opt_cam.h
|
||||
SRCS+= bus_if.h
|
||||
SRCS+= device_if.h
|
||||
SRCS+= icl_conn_if.h
|
||||
|
||||
CFLAGS+= -I${.CURDIR}/../../
|
||||
CFLAGS+= -I${SYSDIR}/ofed/include
|
||||
CFLAGS+= -I${SYSDIR}/compat/linuxkpi/common/include
|
||||
CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM
|
||||
CFLAGS+= -DINET6 -DINET
|
||||
CFLAGS+= -fms-extensions
|
||||
|
||||
CFLAGS+=-DICL_KERNEL_PROXY
|
||||
|
||||
MFILES= kern/bus_if.m kern/device_if.m dev/iscsi/icl_conn_if.m
|
||||
|
||||
.include <bsd.kmod.mk>
|
Loading…
Reference in New Issue
Block a user