538 lines
14 KiB
C
538 lines
14 KiB
C
/*-
|
|
* Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
* SUCH DAMAGE.
|
|
*/
|
|
|
|
#include "icl_iser.h"
|
|
|
|
static MALLOC_DEFINE(M_ISER_INITIATOR, "iser_initiator", "iser initiator backend");
|
|
|
|
/* Register user buffer memory and initialize passive rdma
|
|
* dto descriptor. Data size is stored in
|
|
* task->data[ISER_DIR_IN].data_len, Protection size
|
|
* os stored in task->prot[ISER_DIR_IN].data_len
|
|
*/
|
|
static int
|
|
iser_prepare_read_cmd(struct icl_iser_pdu *iser_pdu)
|
|
{
|
|
struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
|
|
struct iser_data_buf *buf_in = &iser_pdu->data[ISER_DIR_IN];
|
|
struct iser_mem_reg *mem_reg;
|
|
int err;
|
|
|
|
err = iser_dma_map_task_data(iser_pdu,
|
|
buf_in,
|
|
ISER_DIR_IN,
|
|
DMA_FROM_DEVICE);
|
|
if (err)
|
|
return (err);
|
|
|
|
err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_IN);
|
|
if (err) {
|
|
ISER_ERR("Failed to set up Data-IN RDMA");
|
|
return (err);
|
|
}
|
|
|
|
mem_reg = &iser_pdu->rdma_reg[ISER_DIR_IN];
|
|
|
|
hdr->flags |= ISER_RSV;
|
|
hdr->read_stag = cpu_to_be32(mem_reg->rkey);
|
|
hdr->read_va = cpu_to_be64(mem_reg->sge.addr);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/* Register user buffer memory and initialize passive rdma
|
|
* dto descriptor. Data size is stored in
|
|
* task->data[ISER_DIR_OUT].data_len, Protection size
|
|
* is stored at task->prot[ISER_DIR_OUT].data_len
|
|
*/
|
|
static int
|
|
iser_prepare_write_cmd(struct icl_iser_pdu *iser_pdu)
|
|
{
|
|
struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
|
|
struct iser_data_buf *buf_out = &iser_pdu->data[ISER_DIR_OUT];
|
|
struct iser_mem_reg *mem_reg;
|
|
int err;
|
|
|
|
err = iser_dma_map_task_data(iser_pdu,
|
|
buf_out,
|
|
ISER_DIR_OUT,
|
|
DMA_TO_DEVICE);
|
|
if (err)
|
|
return (err);
|
|
|
|
err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_OUT);
|
|
if (err) {
|
|
ISER_ERR("Failed to set up Data-out RDMA");
|
|
return (err);
|
|
}
|
|
|
|
mem_reg = &iser_pdu->rdma_reg[ISER_DIR_OUT];
|
|
|
|
hdr->flags |= ISER_WSV;
|
|
hdr->write_stag = cpu_to_be32(mem_reg->rkey);
|
|
hdr->write_va = cpu_to_be64(mem_reg->sge.addr);
|
|
|
|
return (0);
|
|
}
|
|
|
|
/* creates a new tx descriptor and adds header regd buffer */
|
|
void
|
|
iser_create_send_desc(struct iser_conn *iser_conn,
|
|
struct iser_tx_desc *tx_desc)
|
|
{
|
|
struct iser_device *device = iser_conn->ib_conn.device;
|
|
|
|
ib_dma_sync_single_for_cpu(device->ib_device,
|
|
tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);
|
|
|
|
memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
|
|
tx_desc->iser_header.flags = ISER_VER;
|
|
|
|
tx_desc->num_sge = 1;
|
|
|
|
if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
|
|
tx_desc->tx_sg[0].lkey = device->mr->lkey;
|
|
ISER_DBG("sdesc %p lkey mismatch, fixing", tx_desc);
|
|
}
|
|
}
|
|
|
|
void
|
|
iser_free_login_buf(struct iser_conn *iser_conn)
|
|
{
|
|
struct iser_device *device = iser_conn->ib_conn.device;
|
|
|
|
if (!iser_conn->login_buf)
|
|
return;
|
|
|
|
if (iser_conn->login_req_dma)
|
|
ib_dma_unmap_single(device->ib_device,
|
|
iser_conn->login_req_dma,
|
|
ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);
|
|
|
|
if (iser_conn->login_resp_dma)
|
|
ib_dma_unmap_single(device->ib_device,
|
|
iser_conn->login_resp_dma,
|
|
ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);
|
|
|
|
free(iser_conn->login_buf, M_ISER_INITIATOR);
|
|
|
|
/* make sure we never redo any unmapping */
|
|
iser_conn->login_req_dma = 0;
|
|
iser_conn->login_resp_dma = 0;
|
|
iser_conn->login_buf = NULL;
|
|
}
|
|
|
|
int
|
|
iser_alloc_login_buf(struct iser_conn *iser_conn)
|
|
{
|
|
struct iser_device *device = iser_conn->ib_conn.device;
|
|
int req_err, resp_err;
|
|
|
|
BUG_ON(device == NULL);
|
|
|
|
iser_conn->login_buf = malloc(ISCSI_DEF_MAX_RECV_SEG_LEN + ISER_RX_LOGIN_SIZE,
|
|
M_ISER_INITIATOR, M_WAITOK | M_ZERO);
|
|
|
|
if (!iser_conn->login_buf)
|
|
goto out_err;
|
|
|
|
iser_conn->login_req_buf = iser_conn->login_buf;
|
|
iser_conn->login_resp_buf = iser_conn->login_buf +
|
|
ISCSI_DEF_MAX_RECV_SEG_LEN;
|
|
|
|
iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
|
|
iser_conn->login_req_buf,
|
|
ISCSI_DEF_MAX_RECV_SEG_LEN,
|
|
DMA_TO_DEVICE);
|
|
|
|
iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
|
|
iser_conn->login_resp_buf,
|
|
ISER_RX_LOGIN_SIZE,
|
|
DMA_FROM_DEVICE);
|
|
|
|
req_err = ib_dma_mapping_error(device->ib_device,
|
|
iser_conn->login_req_dma);
|
|
resp_err = ib_dma_mapping_error(device->ib_device,
|
|
iser_conn->login_resp_dma);
|
|
|
|
if (req_err || resp_err) {
|
|
if (req_err)
|
|
iser_conn->login_req_dma = 0;
|
|
if (resp_err)
|
|
iser_conn->login_resp_dma = 0;
|
|
goto free_login_buf;
|
|
}
|
|
|
|
return (0);
|
|
|
|
free_login_buf:
|
|
iser_free_login_buf(iser_conn);
|
|
|
|
out_err:
|
|
ISER_DBG("unable to alloc or map login buf");
|
|
return (ENOMEM);
|
|
}
|
|
|
|
int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, int cmds_max)
|
|
{
|
|
int i, j;
|
|
u64 dma_addr;
|
|
struct iser_rx_desc *rx_desc;
|
|
struct ib_sge *rx_sg;
|
|
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
|
struct iser_device *device = ib_conn->device;
|
|
|
|
iser_conn->qp_max_recv_dtos = cmds_max;
|
|
iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;
|
|
|
|
if (iser_create_fastreg_pool(ib_conn, cmds_max))
|
|
goto create_rdma_reg_res_failed;
|
|
|
|
|
|
iser_conn->num_rx_descs = cmds_max;
|
|
iser_conn->rx_descs = malloc(iser_conn->num_rx_descs *
|
|
sizeof(struct iser_rx_desc), M_ISER_INITIATOR,
|
|
M_WAITOK | M_ZERO);
|
|
if (!iser_conn->rx_descs)
|
|
goto rx_desc_alloc_fail;
|
|
|
|
rx_desc = iser_conn->rx_descs;
|
|
|
|
for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) {
|
|
dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
|
|
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
|
if (ib_dma_mapping_error(device->ib_device, dma_addr))
|
|
goto rx_desc_dma_map_failed;
|
|
|
|
rx_desc->dma_addr = dma_addr;
|
|
|
|
rx_sg = &rx_desc->rx_sg;
|
|
rx_sg->addr = rx_desc->dma_addr;
|
|
rx_sg->length = ISER_RX_PAYLOAD_SIZE;
|
|
rx_sg->lkey = device->mr->lkey;
|
|
}
|
|
|
|
iser_conn->rx_desc_head = 0;
|
|
|
|
return (0);
|
|
|
|
rx_desc_dma_map_failed:
|
|
rx_desc = iser_conn->rx_descs;
|
|
for (j = 0; j < i; j++, rx_desc++)
|
|
ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
|
|
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
|
free(iser_conn->rx_descs, M_ISER_INITIATOR);
|
|
iser_conn->rx_descs = NULL;
|
|
rx_desc_alloc_fail:
|
|
iser_free_fastreg_pool(ib_conn);
|
|
create_rdma_reg_res_failed:
|
|
ISER_ERR("failed allocating rx descriptors / data buffers");
|
|
|
|
return (ENOMEM);
|
|
}
|
|
|
|
void
|
|
iser_free_rx_descriptors(struct iser_conn *iser_conn)
|
|
{
|
|
int i;
|
|
struct iser_rx_desc *rx_desc;
|
|
struct ib_conn *ib_conn = &iser_conn->ib_conn;
|
|
struct iser_device *device = ib_conn->device;
|
|
|
|
iser_free_fastreg_pool(ib_conn);
|
|
|
|
rx_desc = iser_conn->rx_descs;
|
|
for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
|
|
ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
|
|
ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
|
|
|
|
free(iser_conn->rx_descs, M_ISER_INITIATOR);
|
|
|
|
/* make sure we never redo any unmapping */
|
|
iser_conn->rx_descs = NULL;
|
|
}
|
|
|
|
static void
|
|
iser_buf_to_sg(void *buf, struct iser_data_buf *data_buf)
|
|
{
|
|
struct scatterlist *sg;
|
|
int i;
|
|
size_t len, tlen;
|
|
int offset;
|
|
|
|
tlen = data_buf->data_len;
|
|
|
|
for (i = 0; 0 < tlen; i++, tlen -= len) {
|
|
sg = &data_buf->sgl[i];
|
|
offset = ((uintptr_t)buf) & ~PAGE_MASK;
|
|
len = min(PAGE_SIZE - offset, tlen);
|
|
sg_set_buf(sg, buf, len);
|
|
buf = (void *)((uintptr_t)buf + len);
|
|
}
|
|
|
|
data_buf->size = i;
|
|
sg_mark_end(sg);
|
|
}
|
|
|
|
|
|
static void
|
|
iser_bio_to_sg(struct bio *bp, struct iser_data_buf *data_buf)
|
|
{
|
|
struct scatterlist *sg;
|
|
int i;
|
|
size_t len, tlen;
|
|
int offset;
|
|
|
|
tlen = bp->bio_bcount;
|
|
offset = bp->bio_ma_offset;
|
|
|
|
for (i = 0; 0 < tlen; i++, tlen -= len) {
|
|
sg = &data_buf->sgl[i];
|
|
len = min(PAGE_SIZE - offset, tlen);
|
|
sg_set_page(sg, bp->bio_ma[i], len, offset);
|
|
offset = 0;
|
|
}
|
|
|
|
data_buf->size = i;
|
|
sg_mark_end(sg);
|
|
}
|
|
|
|
static int
|
|
iser_csio_to_sg(struct ccb_scsiio *csio, struct iser_data_buf *data_buf)
|
|
{
|
|
struct ccb_hdr *ccbh;
|
|
int err = 0;
|
|
|
|
ccbh = &csio->ccb_h;
|
|
switch ((ccbh->flags & CAM_DATA_MASK)) {
|
|
case CAM_DATA_BIO:
|
|
iser_bio_to_sg((struct bio *) csio->data_ptr, data_buf);
|
|
break;
|
|
case CAM_DATA_VADDR:
|
|
/*
|
|
* Support KVA buffers for various scsi commands such as:
|
|
* - REPORT_LUNS
|
|
* - MODE_SENSE_6
|
|
* - INQUIRY
|
|
* - SERVICE_ACTION_IN.
|
|
* The data of these commands always mapped into KVA.
|
|
*/
|
|
iser_buf_to_sg(csio->data_ptr, data_buf);
|
|
break;
|
|
default:
|
|
ISER_ERR("flags 0x%X unimplemented", ccbh->flags);
|
|
err = EINVAL;
|
|
}
|
|
return (err);
|
|
}
|
|
|
|
static inline bool
|
|
iser_signal_comp(u8 sig_count)
|
|
{
|
|
return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);
|
|
}
|
|
|
|
int
|
|
iser_send_command(struct iser_conn *iser_conn,
|
|
struct icl_iser_pdu *iser_pdu)
|
|
{
|
|
struct iser_data_buf *data_buf;
|
|
struct iser_tx_desc *tx_desc = &iser_pdu->desc;
|
|
struct iscsi_bhs_scsi_command *hdr = (struct iscsi_bhs_scsi_command *) &(iser_pdu->desc.iscsi_header);
|
|
struct ccb_scsiio *csio = iser_pdu->csio;
|
|
int err = 0;
|
|
u8 sig_count = ++iser_conn->ib_conn.sig_count;
|
|
|
|
/* build the tx desc regd header and add it to the tx desc dto */
|
|
tx_desc->type = ISCSI_TX_SCSI_COMMAND;
|
|
iser_create_send_desc(iser_conn, tx_desc);
|
|
|
|
if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
|
|
data_buf = &iser_pdu->data[ISER_DIR_IN];
|
|
} else {
|
|
data_buf = &iser_pdu->data[ISER_DIR_OUT];
|
|
}
|
|
|
|
data_buf->sg = csio->data_ptr;
|
|
data_buf->data_len = csio->dxfer_len;
|
|
|
|
if (likely(csio->dxfer_len)) {
|
|
err = iser_csio_to_sg(csio, data_buf);
|
|
if (unlikely(err))
|
|
goto send_command_error;
|
|
}
|
|
|
|
if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
|
|
err = iser_prepare_read_cmd(iser_pdu);
|
|
if (err)
|
|
goto send_command_error;
|
|
} else if (hdr->bhssc_flags & BHSSC_FLAGS_W) {
|
|
err = iser_prepare_write_cmd(iser_pdu);
|
|
if (err)
|
|
goto send_command_error;
|
|
}
|
|
|
|
err = iser_post_send(&iser_conn->ib_conn, tx_desc,
|
|
iser_signal_comp(sig_count));
|
|
if (!err)
|
|
return (0);
|
|
|
|
send_command_error:
|
|
ISER_ERR("iser_conn %p itt %u len %u err %d", iser_conn,
|
|
hdr->bhssc_initiator_task_tag,
|
|
hdr->bhssc_expected_data_transfer_length,
|
|
err);
|
|
return (err);
|
|
}
|
|
|
|
int
|
|
iser_send_control(struct iser_conn *iser_conn,
|
|
struct icl_iser_pdu *iser_pdu)
|
|
{
|
|
struct iser_tx_desc *mdesc;
|
|
struct iser_device *device;
|
|
size_t datalen = iser_pdu->icl_pdu.ip_data_len;
|
|
int err;
|
|
|
|
mdesc = &iser_pdu->desc;
|
|
|
|
/* build the tx desc regd header and add it to the tx desc dto */
|
|
mdesc->type = ISCSI_TX_CONTROL;
|
|
iser_create_send_desc(iser_conn, mdesc);
|
|
|
|
device = iser_conn->ib_conn.device;
|
|
|
|
if (datalen > 0) {
|
|
struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
|
|
ib_dma_sync_single_for_cpu(device->ib_device,
|
|
iser_conn->login_req_dma, datalen,
|
|
DMA_TO_DEVICE);
|
|
|
|
ib_dma_sync_single_for_device(device->ib_device,
|
|
iser_conn->login_req_dma, datalen,
|
|
DMA_TO_DEVICE);
|
|
|
|
tx_dsg->addr = iser_conn->login_req_dma;
|
|
tx_dsg->length = datalen;
|
|
tx_dsg->lkey = device->mr->lkey;
|
|
mdesc->num_sge = 2;
|
|
}
|
|
|
|
/* For login phase and discovery session we re-use the login buffer */
|
|
if (!iser_conn->handoff_done) {
|
|
err = iser_post_recvl(iser_conn);
|
|
if (err)
|
|
goto send_control_error;
|
|
}
|
|
|
|
err = iser_post_send(&iser_conn->ib_conn, mdesc, true);
|
|
if (!err)
|
|
return (0);
|
|
|
|
send_control_error:
|
|
ISER_ERR("conn %p failed err %d", iser_conn, err);
|
|
|
|
return (err);
|
|
|
|
}
|
|
|
|
/**
|
|
* iser_rcv_dto_completion - recv DTO completion
|
|
*/
|
|
void
|
|
iser_rcv_completion(struct iser_rx_desc *rx_desc,
|
|
unsigned long rx_xfer_len,
|
|
struct ib_conn *ib_conn)
|
|
{
|
|
struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
|
|
ib_conn);
|
|
struct icl_conn *ic = &iser_conn->icl_conn;
|
|
struct icl_pdu *response;
|
|
struct iscsi_bhs *hdr;
|
|
u64 rx_dma;
|
|
int rx_buflen;
|
|
int outstanding, count, err;
|
|
|
|
/* differentiate between login to all other PDUs */
|
|
if ((char *)rx_desc == iser_conn->login_resp_buf) {
|
|
rx_dma = iser_conn->login_resp_dma;
|
|
rx_buflen = ISER_RX_LOGIN_SIZE;
|
|
} else {
|
|
rx_dma = rx_desc->dma_addr;
|
|
rx_buflen = ISER_RX_PAYLOAD_SIZE;
|
|
}
|
|
|
|
ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
|
|
rx_buflen, DMA_FROM_DEVICE);
|
|
|
|
hdr = &rx_desc->iscsi_header;
|
|
|
|
response = iser_new_pdu(ic, M_NOWAIT);
|
|
response->ip_bhs = hdr;
|
|
response->ip_data_len = rx_xfer_len - ISER_HEADERS_LEN;
|
|
|
|
/*
|
|
* In case we got data in the receive buffer, assign the ip_data_mbuf
|
|
* to the rx_buffer - later we'll copy it to upper layer buffers
|
|
*/
|
|
if (response->ip_data_len)
|
|
response->ip_data_mbuf = (struct mbuf *)(rx_desc->data);
|
|
|
|
ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
|
|
rx_buflen, DMA_FROM_DEVICE);
|
|
|
|
/* decrementing conn->post_recv_buf_count only --after-- freeing the *
|
|
* task eliminates the need to worry on tasks which are completed in *
|
|
* parallel to the execution of iser_conn_term. So the code that waits *
|
|
* for the posted rx bufs refcount to become zero handles everything */
|
|
ib_conn->post_recv_buf_count--;
|
|
|
|
if (rx_dma == iser_conn->login_resp_dma)
|
|
goto receive;
|
|
|
|
outstanding = ib_conn->post_recv_buf_count;
|
|
if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
|
|
count = min(iser_conn->qp_max_recv_dtos - outstanding,
|
|
iser_conn->min_posted_rx);
|
|
err = iser_post_recvm(iser_conn, count);
|
|
if (err)
|
|
ISER_ERR("posting %d rx bufs err %d", count, err);
|
|
}
|
|
|
|
receive:
|
|
(ic->ic_receive)(response);
|
|
}
|
|
|
|
void
|
|
iser_snd_completion(struct iser_tx_desc *tx_desc,
|
|
struct ib_conn *ib_conn)
|
|
{
|
|
struct icl_iser_pdu *iser_pdu = container_of(tx_desc, struct icl_iser_pdu, desc);
|
|
struct iser_conn *iser_conn = iser_pdu->iser_conn;
|
|
|
|
if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL)
|
|
iser_pdu_free(&iser_conn->icl_conn, &iser_pdu->icl_pdu);
|
|
}
|