freebsd-skq/sys/dev/iser/iser_initiator.c

/* $FreeBSD$ */
/*-
 * Copyright (c) 2015, Mellanox Technologies, Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include "icl_iser.h"

static MALLOC_DEFINE(M_ISER_INITIATOR, "iser_initiator", "iser initiator backend");

/* Register user buffer memory and initialize passive rdma
 *  dto descriptor. Data size is stored in
 *  task->data[ISER_DIR_IN].data_len, Protection size
 *  os stored in task->prot[ISER_DIR_IN].data_len
 */
static int
iser_prepare_read_cmd(struct icl_iser_pdu *iser_pdu)
{
	struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
	struct iser_data_buf *buf_in = &iser_pdu->data[ISER_DIR_IN];
	struct iser_mem_reg *mem_reg;
	int err;

	err = iser_dma_map_task_data(iser_pdu,
				     buf_in,
				     ISER_DIR_IN,
				     DMA_FROM_DEVICE);
	if (err)
		return (err);

	err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_IN);
	if (err) {
		ISER_ERR("Failed to set up Data-IN RDMA");
		return (err);
	}

	mem_reg = &iser_pdu->rdma_reg[ISER_DIR_IN];

	hdr->flags    |= ISER_RSV;
	hdr->read_stag = cpu_to_be32(mem_reg->rkey);
	hdr->read_va   = cpu_to_be64(mem_reg->sge.addr);

	return (0);
}

/* Register user buffer memory and initialize passive rdma
 *  dto descriptor. Data size is stored in
 *  task->data[ISER_DIR_OUT].data_len, Protection size
 *  is stored at task->prot[ISER_DIR_OUT].data_len
 */
static int
iser_prepare_write_cmd(struct icl_iser_pdu *iser_pdu)
{
	struct iser_hdr *hdr = &iser_pdu->desc.iser_header;
	struct iser_data_buf *buf_out = &iser_pdu->data[ISER_DIR_OUT];
	struct iser_mem_reg *mem_reg;
	int err;

	err = iser_dma_map_task_data(iser_pdu,
				     buf_out,
				     ISER_DIR_OUT,
				     DMA_TO_DEVICE);
	if (err)
		return (err);

	err = iser_reg_rdma_mem(iser_pdu, ISER_DIR_OUT);
	if (err) {
		ISER_ERR("Failed to set up Data-out RDMA");
		return (err);
	}

	mem_reg = &iser_pdu->rdma_reg[ISER_DIR_OUT];

	hdr->flags     |= ISER_WSV;
	hdr->write_stag = cpu_to_be32(mem_reg->rkey);
	hdr->write_va   = cpu_to_be64(mem_reg->sge.addr);

	return (0);
}

/* creates a new tx descriptor and adds header regd buffer */
void
iser_create_send_desc(struct iser_conn *iser_conn,
		      struct iser_tx_desc *tx_desc)
{
	struct iser_device *device = iser_conn->ib_conn.device;

	ib_dma_sync_single_for_cpu(device->ib_device,
		tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE);

	memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr));
	tx_desc->iser_header.flags = ISER_VER;

	tx_desc->num_sge = 1;

	if (tx_desc->tx_sg[0].lkey != device->mr->lkey) {
		tx_desc->tx_sg[0].lkey = device->mr->lkey;
		ISER_DBG("sdesc %p lkey mismatch, fixing", tx_desc);
	}
}

void
iser_free_login_buf(struct iser_conn *iser_conn)
{
	struct iser_device *device = iser_conn->ib_conn.device;

	if (!iser_conn->login_buf)
		return;

	if (iser_conn->login_req_dma)
		ib_dma_unmap_single(device->ib_device,
				    iser_conn->login_req_dma,
				    ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE);

	if (iser_conn->login_resp_dma)
		ib_dma_unmap_single(device->ib_device,
				    iser_conn->login_resp_dma,
				    ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE);

	free(iser_conn->login_buf, M_ISER_INITIATOR);

	/* make sure we never redo any unmapping */
	iser_conn->login_req_dma = 0;
	iser_conn->login_resp_dma = 0;
	iser_conn->login_buf = NULL;
}

int
iser_alloc_login_buf(struct iser_conn *iser_conn)
{
	struct iser_device *device = iser_conn->ib_conn.device;
	int req_err, resp_err;

	BUG_ON(device == NULL);

	iser_conn->login_buf = malloc(ISCSI_DEF_MAX_RECV_SEG_LEN + ISER_RX_LOGIN_SIZE,
				      M_ISER_INITIATOR, M_WAITOK | M_ZERO);

	if (!iser_conn->login_buf)
		goto out_err;

	iser_conn->login_req_buf  = iser_conn->login_buf;
	iser_conn->login_resp_buf = iser_conn->login_buf +
				    ISCSI_DEF_MAX_RECV_SEG_LEN;

	iser_conn->login_req_dma = ib_dma_map_single(device->ib_device,
						     iser_conn->login_req_buf,
						     ISCSI_DEF_MAX_RECV_SEG_LEN,
						     DMA_TO_DEVICE);

	iser_conn->login_resp_dma = ib_dma_map_single(device->ib_device,
						      iser_conn->login_resp_buf,
						      ISER_RX_LOGIN_SIZE,
						      DMA_FROM_DEVICE);

	req_err  = ib_dma_mapping_error(device->ib_device,
					iser_conn->login_req_dma);
	resp_err = ib_dma_mapping_error(device->ib_device,
					iser_conn->login_resp_dma);

	if (req_err || resp_err) {
		if (req_err)
			iser_conn->login_req_dma = 0;
		if (resp_err)
			iser_conn->login_resp_dma = 0;
		goto free_login_buf;
	}

	return (0);

free_login_buf:
	iser_free_login_buf(iser_conn);

out_err:
	ISER_DBG("unable to alloc or map login buf");
	return (ENOMEM);
}

int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, int cmds_max)
{
	int i, j;
	u64 dma_addr;
	struct iser_rx_desc *rx_desc;
	struct ib_sge       *rx_sg;
	struct ib_conn *ib_conn = &iser_conn->ib_conn;
	struct iser_device *device = ib_conn->device;

	iser_conn->qp_max_recv_dtos = cmds_max;
	iser_conn->min_posted_rx = iser_conn->qp_max_recv_dtos >> 2;

	if (iser_create_fastreg_pool(ib_conn, cmds_max))
		goto create_rdma_reg_res_failed;


	iser_conn->num_rx_descs = cmds_max;
	iser_conn->rx_descs = malloc(iser_conn->num_rx_descs *
				sizeof(struct iser_rx_desc), M_ISER_INITIATOR,
				M_WAITOK | M_ZERO);
	if (!iser_conn->rx_descs)
		goto rx_desc_alloc_fail;

	rx_desc = iser_conn->rx_descs;

	for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)  {
		dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc,
					ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
		if (ib_dma_mapping_error(device->ib_device, dma_addr))
			goto rx_desc_dma_map_failed;

		rx_desc->dma_addr = dma_addr;

		rx_sg = &rx_desc->rx_sg;
		rx_sg->addr   = rx_desc->dma_addr;
		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
		rx_sg->lkey   = device->mr->lkey;
	}

	iser_conn->rx_desc_head = 0;

	return (0);

rx_desc_dma_map_failed:
	rx_desc = iser_conn->rx_descs;
	for (j = 0; j < i; j++, rx_desc++)
		ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
	free(iser_conn->rx_descs, M_ISER_INITIATOR);
	iser_conn->rx_descs = NULL;
rx_desc_alloc_fail:
	iser_free_fastreg_pool(ib_conn);
create_rdma_reg_res_failed:
	ISER_ERR("failed allocating rx descriptors / data buffers");

	return (ENOMEM);
}

void
iser_free_rx_descriptors(struct iser_conn *iser_conn)
{
	int i;
	struct iser_rx_desc *rx_desc;
	struct ib_conn *ib_conn = &iser_conn->ib_conn;
	struct iser_device *device = ib_conn->device;

	iser_free_fastreg_pool(ib_conn);

	rx_desc = iser_conn->rx_descs;
	for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++)
		ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr,
				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);

	free(iser_conn->rx_descs, M_ISER_INITIATOR);

	/* make sure we never redo any unmapping */
	iser_conn->rx_descs = NULL;
}

static void
iser_buf_to_sg(void *buf, struct iser_data_buf *data_buf)
{
	struct scatterlist *sg;
	int i;
	size_t len, tlen;
	int offset;

	tlen = data_buf->data_len;

	for (i = 0; 0 < tlen; i++, tlen -= len)  {
		sg = &data_buf->sgl[i];
		offset = ((uintptr_t)buf) & ~PAGE_MASK;
		len = min(PAGE_SIZE - offset, tlen);
		sg_set_buf(sg, buf, len);
		buf = (void *)(((u64)buf) + (u64)len);
	}

	data_buf->size = i;
	sg_mark_end(sg);
}


static void
iser_bio_to_sg(struct bio *bp, struct iser_data_buf *data_buf)
{
	struct scatterlist *sg;
	int i;
	size_t len, tlen;
	int offset;

	tlen = bp->bio_bcount;
	offset = bp->bio_ma_offset;

	for (i = 0; 0 < tlen; i++, tlen -= len) {
		sg = &data_buf->sgl[i];
		len = min(PAGE_SIZE - offset, tlen);
		sg_set_page(sg, bp->bio_ma[i], len, offset);
		offset = 0;
	}

	data_buf->size = i;
	sg_mark_end(sg);
}

static int
iser_csio_to_sg(struct ccb_scsiio *csio, struct iser_data_buf *data_buf)
{
	struct ccb_hdr *ccbh;
	int err = 0;

	ccbh = &csio->ccb_h;
	switch ((ccbh->flags & CAM_DATA_MASK)) {
		case CAM_DATA_BIO:
			iser_bio_to_sg((struct bio *) csio->data_ptr, data_buf);
			break;
		case CAM_DATA_VADDR:
			/*
			 * Support KVA buffers for various scsi commands such as:
			 *  - REPORT_LUNS
			 *  - MODE_SENSE_6
			 *  - INQUIRY
			 *  - SERVICE_ACTION_IN.
			 * The data of these commands always mapped into KVA.
			 */
			iser_buf_to_sg(csio->data_ptr, data_buf);
			break;
		default:
			ISER_ERR("flags 0x%X unimplemented", ccbh->flags);
			err = EINVAL;
	}
	return (err);
}

static inline bool
iser_signal_comp(u8 sig_count)
{
	return ((sig_count % ISER_SIGNAL_CMD_COUNT) == 0);
}

int
iser_send_command(struct iser_conn *iser_conn,
		  struct icl_iser_pdu *iser_pdu)
{
	struct iser_data_buf *data_buf;
	struct iser_tx_desc *tx_desc = &iser_pdu->desc;
	struct iscsi_bhs_scsi_command *hdr = (struct iscsi_bhs_scsi_command *) &(iser_pdu->desc.iscsi_header);
	struct ccb_scsiio *csio = iser_pdu->csio;
	int err = 0;
	u8 sig_count = ++iser_conn->ib_conn.sig_count;

	/* build the tx desc regd header and add it to the tx desc dto */
	tx_desc->type = ISCSI_TX_SCSI_COMMAND;
	iser_create_send_desc(iser_conn, tx_desc);

	if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
		data_buf = &iser_pdu->data[ISER_DIR_IN];
	} else {
		data_buf = &iser_pdu->data[ISER_DIR_OUT];
	}

	data_buf->sg = csio->data_ptr;
	data_buf->data_len = csio->dxfer_len;

	if (likely(csio->dxfer_len)) {
		err = iser_csio_to_sg(csio, data_buf);
		if (unlikely(err))
			goto send_command_error;
	}

	if (hdr->bhssc_flags & BHSSC_FLAGS_R) {
		err = iser_prepare_read_cmd(iser_pdu);
		if (err)
			goto send_command_error;
	} else if (hdr->bhssc_flags & BHSSC_FLAGS_W) {
		err = iser_prepare_write_cmd(iser_pdu);
		if (err)
			goto send_command_error;
	}

	err = iser_post_send(&iser_conn->ib_conn, tx_desc,
			     iser_signal_comp(sig_count));
	if (!err)
		return (0);

send_command_error:
	ISER_ERR("iser_conn %p itt %u len %u err %d", iser_conn,
			hdr->bhssc_initiator_task_tag,
			hdr->bhssc_expected_data_transfer_length,
			err);
	return (err);
}

int
iser_send_control(struct iser_conn *iser_conn,
		  struct icl_iser_pdu *iser_pdu)
{
	struct iser_tx_desc *mdesc;
	struct iser_device *device;
	size_t datalen = iser_pdu->icl_pdu.ip_data_len;
	int err;

	mdesc = &iser_pdu->desc;

	/* build the tx desc regd header and add it to the tx desc dto */
	mdesc->type = ISCSI_TX_CONTROL;
	iser_create_send_desc(iser_conn, mdesc);

	device = iser_conn->ib_conn.device;

	if (datalen > 0) {
		struct ib_sge *tx_dsg = &mdesc->tx_sg[1];
		ib_dma_sync_single_for_cpu(device->ib_device,
				iser_conn->login_req_dma, datalen,
				DMA_TO_DEVICE);

		ib_dma_sync_single_for_device(device->ib_device,
			iser_conn->login_req_dma, datalen,
			DMA_TO_DEVICE);

		tx_dsg->addr    = iser_conn->login_req_dma;
		tx_dsg->length  = datalen;
		tx_dsg->lkey    = device->mr->lkey;
		mdesc->num_sge = 2;
	}

	/* For login phase and discovery session we re-use the login buffer */
	if (!iser_conn->handoff_done) {
		err = iser_post_recvl(iser_conn);
		if (err)
			goto send_control_error;
	}

	err = iser_post_send(&iser_conn->ib_conn, mdesc, true);
	if (!err)
		return (0);

send_control_error:
	ISER_ERR("conn %p failed err %d", iser_conn, err);

	return (err);

}

/**
 * iser_rcv_dto_completion - recv DTO completion
 */
void
iser_rcv_completion(struct iser_rx_desc *rx_desc,
		    unsigned long rx_xfer_len,
		    struct ib_conn *ib_conn)
{
	struct iser_conn *iser_conn = container_of(ib_conn, struct iser_conn,
						   ib_conn);
	struct icl_conn *ic = &iser_conn->icl_conn;
	struct icl_pdu *response;
	struct iscsi_bhs *hdr;
	u64 rx_dma;
	int rx_buflen;
	int outstanding, count, err;

	/* differentiate between login to all other PDUs */
	if ((char *)rx_desc == iser_conn->login_resp_buf) {
		rx_dma = iser_conn->login_resp_dma;
		rx_buflen = ISER_RX_LOGIN_SIZE;
	} else {
		rx_dma = rx_desc->dma_addr;
		rx_buflen = ISER_RX_PAYLOAD_SIZE;
	}

	ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma,
				   rx_buflen, DMA_FROM_DEVICE);

	hdr = &rx_desc->iscsi_header;

	response = iser_new_pdu(ic, M_NOWAIT);
	response->ip_bhs = hdr;
	response->ip_data_len = rx_xfer_len - ISER_HEADERS_LEN;

	/*
	 * In case we got data in the receive buffer, assign the ip_data_mbuf
	 * to the rx_buffer - later we'll copy it to upper layer buffers
	 */
	if (response->ip_data_len)
		response->ip_data_mbuf = (struct mbuf *)(rx_desc->data);

	ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma,
				      rx_buflen, DMA_FROM_DEVICE);

	/* decrementing conn->post_recv_buf_count only --after-- freeing the   *
	 * task eliminates the need to worry on tasks which are completed in   *
	 * parallel to the execution of iser_conn_term. So the code that waits *
	 * for the posted rx bufs refcount to become zero handles everything   */
	ib_conn->post_recv_buf_count--;

	if (rx_dma == iser_conn->login_resp_dma)
		goto receive;

	outstanding = ib_conn->post_recv_buf_count;
	if (outstanding + iser_conn->min_posted_rx <= iser_conn->qp_max_recv_dtos) {
		count = min(iser_conn->qp_max_recv_dtos - outstanding,
			    iser_conn->min_posted_rx);
		err = iser_post_recvm(iser_conn, count);
		if (err)
			ISER_ERR("posting %d rx bufs err %d", count, err);
	}

receive:
	(ic->ic_receive)(response);
}

void
iser_snd_completion(struct iser_tx_desc *tx_desc,
		    struct ib_conn *ib_conn)
{
	struct icl_iser_pdu *iser_pdu = container_of(tx_desc, struct icl_iser_pdu, desc);
	struct iser_conn *iser_conn = iser_pdu->iser_conn;

	if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL)
		iser_pdu_free(&iser_conn->icl_conn, &iser_pdu->icl_pdu);
}