d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the iw_cxgbe bits in the projects branch.

Browse Source 
Submitted by:	Krishnamraju Eraparaju @ Chelsio
Sponsored by:	Chelsio Communications
This commit is contained in:
Navdeep Parhar 2017-11-07 23:52:14 +00:00
parent c2c014f24c
commit 5c2bacde58
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/projects/bsd_rdma_4_9/; revision=325522
21 changed files with 2194 additions and 1234 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/ofed/libcxgb4/cq.c
View File

@ -437,7 +437,7 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
if (!*cqe_flushed && CQE_STATUS(hw_cqe))
dump_cqe(hw_cqe);
BUG_ON((*cqe_flushed == 0) && !SW_CQE(hw_cqe));
BUG_ON((cqe_flushed == 0) && !SW_CQE(hw_cqe));
goto proc_cqe;
}

54
contrib/ofed/libcxgb4/dev.c
View File

@ -39,6 +39,7 @@
#include <pthread.h>
#include <string.h>
#include <signal.h>
#include <stdbool.h>
#include "libcxgb4.h"
#include "cxgb4-abi.h"
@ -194,6 +195,17 @@ static struct ibv_context *c4iw_alloc_context(struct ibv_device *ibdev,
rhp->cqid2ptr = calloc(rhp->max_cq, sizeof(void *));
if (!rhp->cqid2ptr)
goto err_unmap;
/* Disable userspace WC if architecture/adapter does not
* support WC.
* Note: To forcefully disable WC in kernel driver use the
* loader tunable "hw.cxl.write_combine=0"
*/
if (t5_en_wc && !context->status_page->wc_supported) {
fprintf(stderr, "iw_cxgb4 driver doesn't support Write "
"Combine, so regular DB writes will be used\n");
t5_en_wc = 0;
}
}
return &context->ibv_ctx;
@ -400,11 +412,44 @@ int c4iw_abi_version = 1;
static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
int abi_version)
{
char devstr[IBV_SYSFS_PATH_MAX], ibdev[16], value[32], *cp;
char devstr[IBV_SYSFS_PATH_MAX], ibdev[16], value[128], *cp;
char dev_str[IBV_SYSFS_PATH_MAX];
struct c4iw_dev *dev;
unsigned vendor, device, fw_maj, fw_min;
int i;
char devnum;
char ib_param[16];
#ifndef __linux__
if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev",
ibdev, sizeof ibdev) < 0)
return NULL;
devnum = atoi(&ibdev[5]);
if (ibdev[0] == 't' && ibdev[1] >= '4' && ibdev[1] <= '6' &&
strstr(&ibdev[2], "nex") && devnum >= 0) {
snprintf(dev_str, sizeof(dev_str), "/dev/t%cnex/%d", ibdev[1],
devnum);
} else
return NULL;
if (ibv_read_sysfs_file(dev_str, "\%pnpinfo", value, sizeof value) < 0)
return NULL;
else {
if (strstr(value, "vendor=")) {
strncpy(ib_param, strstr(value, "vendor=") +
strlen("vendor="), 6);
sscanf(ib_param, "%i", &vendor);
}
if (strstr(value, "device=")) {
strncpy(ib_param, strstr(value, "device=") +
strlen("device="), 6);
sscanf(ib_param, "%i", &device);
}
}
#else
if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor",
value, sizeof value) < 0)
return NULL;
@ -414,6 +459,7 @@ static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
value, sizeof value) < 0)
return NULL;
sscanf(value, "%i", &device);
#endif
for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i)
if (vendor == hca_table[i].vendor &&
@ -425,6 +471,11 @@ static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
found:
c4iw_abi_version = abi_version;
#ifndef __linux__
if (ibv_read_sysfs_file(dev_str, "firmware_version",
value, sizeof value) < 0)
return NULL;
#else
/*
* Verify that the firmware major number matches. Major number
* mismatches are fatal. Minor number mismatches are tolerated.
@ -438,6 +489,7 @@ static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
ibv_get_sysfs_path(), ibdev);
if (ibv_read_sysfs_file(devstr, "fw_ver", value, sizeof value) < 0)
return NULL;
#endif
cp = strtok(value+1, ".");
sscanf(cp, "%i", &fw_maj);

57
contrib/ofed/libcxgb4/qp.c
View File

@ -44,10 +44,13 @@ struct c4iw_stats c4iw_stats;
static void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16)
{
u64 *src, *dst;
void *src, *dst;
uintptr_t end;
int total, len;
src = (u64 *)wqe;
dst = (u64 *)((u8 *)wq->sq.queue + wq->sq.wq_pidx * T4_EQ_ENTRY_SIZE);
src = &wqe->flits[0];
dst = &wq->sq.queue->flits[wq->sq.wq_pidx *
(T4_EQ_ENTRY_SIZE / sizeof(__be64))];
if (t4_sq_onchip(wq)) {
len16 = align(len16, 4);
@ -57,17 +60,18 @@ static void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16)
* happens */
mmio_wc_start();
}
while (len16) {
*dst++ = *src++;
if (dst == (u64 *)&wq->sq.queue[wq->sq.size])
dst = (u64 *)wq->sq.queue;
*dst++ = *src++;
if (dst == (u64 *)&wq->sq.queue[wq->sq.size])
dst = (u64 *)wq->sq.queue;
len16--;
/* NOTE len16 cannot be large enough to write to the
same sq.queue memory twice in this loop */
/* NOTE len16 cannot be large enough to write to the
same sq.queue memory twice in this loop */
total = len16 * 16;
end = (uintptr_t)&wq->sq.queue[wq->sq.size];
if (__predict_true((uintptr_t)dst + total <= end)) {
/* Won't wrap around. */
memcpy(dst, src, total);
} else {
len = end - (uintptr_t)dst;
memcpy(dst, src, len);
memcpy(wq->sq.queue, src + len, total - len);
}
if (t4_sq_onchip(wq))
@ -76,18 +80,23 @@ static void copy_wr_to_sq(struct t4_wq *wq, union t4_wr *wqe, u8 len16)
static void copy_wr_to_rq(struct t4_wq *wq, union t4_recv_wr *wqe, u8 len16)
{
u64 *src, *dst;
void *src, *dst;
uintptr_t end;
int total, len;
src = (u64 *)wqe;
dst = (u64 *)((u8 *)wq->rq.queue + wq->rq.wq_pidx * T4_EQ_ENTRY_SIZE);
while (len16) {
*dst++ = *src++;
if (dst >= (u64 *)&wq->rq.queue[wq->rq.size])
dst = (u64 *)wq->rq.queue;
*dst++ = *src++;
if (dst >= (u64 *)&wq->rq.queue[wq->rq.size])
dst = (u64 *)wq->rq.queue;
len16--;
src = &wqe->flits[0];
dst = &wq->rq.queue->flits[wq->rq.wq_pidx *
(T4_EQ_ENTRY_SIZE / sizeof(__be64))];
total = len16 * 16;
end = (uintptr_t)&wq->rq.queue[wq->rq.size];
if (__predict_true((uintptr_t)dst + total <= end)) {
/* Won't wrap around. */
memcpy(dst, src, total);
} else {
len = end - (uintptr_t)dst;
memcpy(dst, src, len);
memcpy(wq->rq.queue, src + len, total - len);
}
}

4
contrib/ofed/libcxgb4/t4.h
View File

@ -87,7 +87,7 @@
#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1)
#define T4_MAX_NUM_STAG (1<<15)
#define T4_MAX_MR_SIZE (~0ULL - 1)
#define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */
#define T4_PAGESIZE_MASK 0xffffffff000 /* 4KB-8TB */
#define T4_STAG_UNSET 0xffffffff
#define T4_FW_MAJ 0
@ -723,7 +723,7 @@ static inline void t4_reset_cq_in_error(struct t4_cq *cq)
struct t4_dev_status_page
{
u8 db_off;
u8 pad1;
u8 wc_supported;
u16 pad2;
u32 pad3;
u64 qp_start;

4
contrib/ofed/libcxgb4/verbs.c
View File

@ -468,7 +468,7 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd,
}
qhp->wq.sq.queue = mmap(NULL, qhp->wq.sq.memsize,
PROT_WRITE, MAP_SHARED,
PROT_READ|PROT_WRITE, MAP_SHARED,
pd->context->cmd_fd, resp.sq_key);
if (qhp->wq.sq.queue == MAP_FAILED)
goto err4;
@ -490,7 +490,7 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd,
qhp->wq.rq.udb += 2;
}
qhp->wq.rq.queue = mmap(NULL, qhp->wq.rq.memsize,
PROT_WRITE, MAP_SHARED,
PROT_READ|PROT_WRITE, MAP_SHARED,
pd->context->cmd_fd, resp.rq_key);
if (qhp->wq.rq.queue == MAP_FAILED)
goto err6;

2
sys/compat/linuxkpi/common/include/linux/dma-mapping.h
View File

@ -127,7 +127,7 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
size_t align;
void *mem;
if (dev->dma_mask)
if (dev != NULL && dev->dma_mask)
high = *dev->dma_mask;
else if (flag & GFP_DMA32)
high = BUS_SPACE_MAXADDR_32BIT;

1
sys/dev/cxgbe/adapter.h
View File

@ -801,6 +801,7 @@ struct adapter {
void *tom_softc; /* (struct tom_data *) */
struct tom_tunables tt;
struct iw_tunables iwt;
void *iwarp_softc; /* (struct c4iw_dev *) */
void *iscsi_ulp_softc; /* (struct cxgbei_data *) */
void *ccr_softc; /* (struct ccr_softc *) */

5
sys/dev/cxgbe/common/common.h
View File

@ -68,6 +68,8 @@ enum {
FEC_RESERVED = 1 << 2,
};
enum t4_bar2_qtype { T4_BAR2_QTYPE_EGRESS, T4_BAR2_QTYPE_INGRESS };
struct port_stats {
u64 tx_octets; /* total # of octets in good frames */
u64 tx_frames; /* all good frames */
@ -843,5 +845,8 @@ int t4vf_get_sge_params(struct adapter *adapter);
int t4vf_get_rss_glb_config(struct adapter *adapter);
int t4vf_get_vfres(struct adapter *adapter);
int t4vf_prep_adapter(struct adapter *adapter);
int t4_bar2_sge_qregs(struct adapter *adapter, unsigned int qid,
enum t4_bar2_qtype qtype, int user, u64 *pbar2_qoffset,
unsigned int *pbar2_qid);
#endif /* __CHELSIO_COMMON_H */

92
sys/dev/cxgbe/common/t4_hw.c
View File

@ -8080,6 +8080,98 @@ int t4_shutdown_adapter(struct adapter *adapter)
return 0;
}
/**
* t4_bar2_sge_qregs - return BAR2 SGE Queue register information
* @adapter: the adapter
* @qid: the Queue ID
* @qtype: the Ingress or Egress type for @qid
* @user: true if this request is for a user mode queue
* @pbar2_qoffset: BAR2 Queue Offset
* @pbar2_qid: BAR2 Queue ID or 0 for Queue ID inferred SGE Queues
*
* Returns the BAR2 SGE Queue Registers information associated with the
* indicated Absolute Queue ID. These are passed back in return value
* pointers. @qtype should be T4_BAR2_QTYPE_EGRESS for Egress Queue
* and T4_BAR2_QTYPE_INGRESS for Ingress Queues.
*
* This may return an error which indicates that BAR2 SGE Queue
* registers aren't available. If an error is not returned, then the
* following values are returned:
*
* *@pbar2_qoffset: the BAR2 Offset of the @qid Registers
* *@pbar2_qid: the BAR2 SGE Queue ID or 0 of @qid
*
* If the returned BAR2 Queue ID is 0, then BAR2 SGE registers which
* require the "Inferred Queue ID" ability may be used. E.g. the
* Write Combining Doorbell Buffer. If the BAR2 Queue ID is not 0,
* then these "Inferred Queue ID" register may not be used.
*/
int t4_bar2_sge_qregs(struct adapter *adapter,
unsigned int qid,
enum t4_bar2_qtype qtype,
int user,
u64 *pbar2_qoffset,
unsigned int *pbar2_qid)
{
unsigned int page_shift, page_size, qpp_shift, qpp_mask;
u64 bar2_page_offset, bar2_qoffset;
unsigned int bar2_qid, bar2_qid_offset, bar2_qinferred;
/* T4 doesn't support BAR2 SGE Queue registers for kernel
* mode queues.
*/
if (!user && is_t4(adapter))
return -EINVAL;
/* Get our SGE Page Size parameters.
*/
page_shift = adapter->params.sge.page_shift;
page_size = 1 << page_shift;
/* Get the right Queues per Page parameters for our Queue.
*/
qpp_shift = (qtype == T4_BAR2_QTYPE_EGRESS
? adapter->params.sge.eq_s_qpp
: adapter->params.sge.iq_s_qpp);
qpp_mask = (1 << qpp_shift) - 1;
/* Calculate the basics of the BAR2 SGE Queue register area:
* o The BAR2 page the Queue registers will be in.
* o The BAR2 Queue ID.
* o The BAR2 Queue ID Offset into the BAR2 page.
*/
bar2_page_offset = ((u64)(qid >> qpp_shift) << page_shift);
bar2_qid = qid & qpp_mask;
bar2_qid_offset = bar2_qid * SGE_UDB_SIZE;
/* If the BAR2 Queue ID Offset is less than the Page Size, then the
* hardware will infer the Absolute Queue ID simply from the writes to
* the BAR2 Queue ID Offset within the BAR2 Page (and we need to use a
* BAR2 Queue ID of 0 for those writes). Otherwise, we'll simply
* write to the first BAR2 SGE Queue Area within the BAR2 Page with
* the BAR2 Queue ID and the hardware will infer the Absolute Queue ID
* from the BAR2 Page and BAR2 Queue ID.
*
* One important censequence of this is that some BAR2 SGE registers
* have a "Queue ID" field and we can write the BAR2 SGE Queue ID
* there. But other registers synthesize the SGE Queue ID purely
* from the writes to the registers -- the Write Combined Doorbell
* Buffer is a good example. These BAR2 SGE Registers are only
* available for those BAR2 SGE Register areas where the SGE Absolute
* Queue ID can be inferred from simple writes.
*/
bar2_qoffset = bar2_page_offset;
bar2_qinferred = (bar2_qid_offset < page_size);
if (bar2_qinferred) {
bar2_qoffset += bar2_qid_offset;
bar2_qid = 0;
}
*pbar2_qoffset = bar2_qoffset;
*pbar2_qid = bar2_qid;
return 0;
}
/**
* t4_init_devlog_params - initialize adapter->params.devlog
* @adap: the adapter

1005
sys/dev/cxgbe/iw_cxgbe/cm.c
View File

File diff suppressed because it is too large Load Diff

441
sys/dev/cxgbe/iw_cxgbe/cq.c
View File

@ -53,6 +53,7 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
struct c4iw_dev_ucontext *uctx)
{
struct adapter *sc = rdev->adap;
struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev);
struct fw_ri_res_wr *res_wr;
struct fw_ri_res *res;
int wr_len;
@ -80,10 +81,12 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
t4_wrq_tx(sc, wr);
c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
kfree(cq->sw_queue);
contigfree(cq->queue, cq->memsize, M_DEVBUF);
dma_free_coherent(rhp->ibdev.dma_device,
cq->memsize, cq->queue,
dma_unmap_addr(cq, mapping));
c4iw_put_cqid(rdev, cq->cqid, uctx);
return 0;
}
@ -93,6 +96,7 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
struct c4iw_dev_ucontext *uctx)
{
struct adapter *sc = rdev->adap;
struct c4iw_dev *rhp = rdev_to_c4iw_dev(rdev);
struct fw_ri_res_wr *res_wr;
struct fw_ri_res *res;
int wr_len;
@ -100,6 +104,7 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
struct c4iw_wr_wait wr_wait;
int ret;
struct wrqe *wr;
u64 cq_bar2_qoffset = 0;
cq->cqid = c4iw_get_cqid(rdev, uctx);
if (!cq->cqid) {
@ -114,17 +119,13 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
goto err2;
}
}
cq->queue = contigmalloc(cq->memsize, M_DEVBUF, M_NOWAIT, 0ul, ~0ul,
PAGE_SIZE, 0);
if (cq->queue)
cq->dma_addr = vtophys(cq->queue);
else {
cq->queue = dma_alloc_coherent(rhp->ibdev.dma_device, cq->memsize,
&cq->dma_addr, GFP_KERNEL);
if (!cq->queue) {
ret = -ENOMEM;
goto err3;
goto err3;
}
pci_unmap_addr_set(cq, mapping, cq->dma_addr);
dma_unmap_addr_set(cq, mapping, cq->dma_addr);
memset(cq->queue, 0, cq->memsize);
/* build fw_ri_res_wr */
@ -166,26 +167,30 @@ create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq,
t4_wrq_tx(sc, wr);
CTR2(KTR_IW_CXGBE, "%s wait_event wr_wait %p", __func__, &wr_wait);
ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
if (ret)
goto err4;
cq->gen = 1;
cq->gts = (void *)((unsigned long)rman_get_virtual(sc->regs_res) +
sc->sge_gts_reg);
cq->rdev = rdev;
if (user) {
cq->ugts = (u64)((char*)rman_get_virtual(sc->udbs_res) +
(cq->cqid << rdev->cqshift));
cq->ugts &= PAGE_MASK;
CTR5(KTR_IW_CXGBE,
"%s: UGTS %p cqid %x cqshift %d page_mask %x", __func__,
cq->ugts, cq->cqid, rdev->cqshift, PAGE_MASK);
}
/* Determine the BAR2 queue offset and qid. */
t4_bar2_sge_qregs(rdev->adap, cq->cqid, T4_BAR2_QTYPE_INGRESS, user,
&cq_bar2_qoffset, &cq->bar2_qid);
/* If user mapping then compute the page-aligned physical
* address for mapping.
*/
if (user)
cq->bar2_pa = (rdev->bar2_pa + cq_bar2_qoffset) & PAGE_MASK;
else
cq->bar2_va = (void __iomem *)((u64)rdev->bar2_kva +
cq_bar2_qoffset);
return 0;
err4:
contigfree(cq->queue, cq->memsize, M_DEVBUF);
dma_free_coherent(rhp->ibdev.dma_device, cq->memsize, cq->queue,
dma_unmap_addr(cq, mapping));
err3:
kfree(cq->sw_queue);
err2:
@ -245,43 +250,188 @@ static void insert_sq_cqe(struct t4_wq *wq, struct t4_cq *cq,
t4_swcq_produce(cq);
}
int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count)
static void advance_oldest_read(struct t4_wq *wq);
int c4iw_flush_sq(struct c4iw_qp *qhp)
{
int flushed = 0;
struct t4_swsqe *swsqe = &wq->sq.sw_sq[wq->sq.cidx + count];
int in_use = wq->sq.in_use - count;
struct t4_wq *wq = &qhp->wq;
struct c4iw_cq *chp = to_c4iw_cq(qhp->ibqp.send_cq);
struct t4_cq *cq = &chp->cq;
int idx;
struct t4_swsqe *swsqe;
BUG_ON(in_use < 0);
while (in_use--) {
swsqe->signaled = 0;
if (wq->sq.flush_cidx == -1)
wq->sq.flush_cidx = wq->sq.cidx;
idx = wq->sq.flush_cidx;
BUG_ON(idx >= wq->sq.size);
while (idx != wq->sq.pidx) {
swsqe = &wq->sq.sw_sq[idx];
BUG_ON(swsqe->flushed);
swsqe->flushed = 1;
insert_sq_cqe(wq, cq, swsqe);
swsqe++;
if (swsqe == (wq->sq.sw_sq + wq->sq.size))
swsqe = wq->sq.sw_sq;
if (wq->sq.oldest_read == swsqe) {
BUG_ON(swsqe->opcode != FW_RI_READ_REQ);
advance_oldest_read(wq);
}
flushed++;
if (++idx == wq->sq.size)
idx = 0;
}
wq->sq.flush_cidx += flushed;
if (wq->sq.flush_cidx >= wq->sq.size)
wq->sq.flush_cidx -= wq->sq.size;
return flushed;
}
static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
{
struct t4_swsqe *swsqe;
int cidx;
if (wq->sq.flush_cidx == -1)
wq->sq.flush_cidx = wq->sq.cidx;
cidx = wq->sq.flush_cidx;
BUG_ON(cidx > wq->sq.size);
while (cidx != wq->sq.pidx) {
swsqe = &wq->sq.sw_sq[cidx];
if (!swsqe->signaled) {
if (++cidx == wq->sq.size)
cidx = 0;
} else if (swsqe->complete) {
BUG_ON(swsqe->flushed);
/*
* Insert this completed cqe into the swcq.
*/
CTR3(KTR_IW_CXGBE,
"%s moving cqe into swcq sq idx %u cq idx %u\n",
__func__, cidx, cq->sw_pidx);
swsqe->cqe.header |= htonl(V_CQE_SWCQE(1));
cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
t4_swcq_produce(cq);
swsqe->flushed = 1;
if (++cidx == wq->sq.size)
cidx = 0;
wq->sq.flush_cidx = cidx;
} else
break;
}
}
static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
struct t4_cqe *read_cqe)
{
read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
read_cqe->len = htonl(wq->sq.oldest_read->read_len);
read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) |
V_CQE_SWCQE(SW_CQE(hw_cqe)) |
V_CQE_OPCODE(FW_RI_READ_REQ) |
V_CQE_TYPE(1));
read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
}
static void advance_oldest_read(struct t4_wq *wq)
{
u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
if (rptr == wq->sq.size)
rptr = 0;
while (rptr != wq->sq.pidx) {
wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
return;
if (++rptr == wq->sq.size)
rptr = 0;
}
wq->sq.oldest_read = NULL;
}
/*
* Move all CQEs from the HWCQ into the SWCQ.
* Deal with out-of-order and/or completions that complete
* prior unsignalled WRs.
*/
void c4iw_flush_hw_cq(struct t4_cq *cq)
void c4iw_flush_hw_cq(struct c4iw_cq *chp)
{
struct t4_cqe *cqe = NULL, *swcqe;
struct t4_cqe *hw_cqe, *swcqe, read_cqe;
struct c4iw_qp *qhp;
struct t4_swsqe *swsqe;
int ret;
CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, cq, cq->cqid);
ret = t4_next_hw_cqe(cq, &cqe);
CTR3(KTR_IW_CXGBE, "%s cq %p cqid 0x%x", __func__, &chp->cq,
chp->cq.cqid);
ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
/*
* This logic is similar to poll_cq(), but not quite the same
* unfortunately. Need to move pertinent HW CQEs to the SW CQ but
* also do any translation magic that poll_cq() normally does.
*/
while (!ret) {
CTR3(KTR_IW_CXGBE, "%s flushing hwcq cidx 0x%x swcq pidx 0x%x",
__func__, cq->cidx, cq->sw_pidx);
swcqe = &cq->sw_queue[cq->sw_pidx];
*swcqe = *cqe;
swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1));
t4_swcq_produce(cq);
t4_hwcq_consume(cq);
ret = t4_next_hw_cqe(cq, &cqe);
qhp = get_qhp(chp->rhp, CQE_QPID(hw_cqe));
/*
* drop CQEs with no associated QP
*/
if (qhp == NULL)
goto next_cqe;
if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE)
goto next_cqe;
if (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP) {
/* If we have reached here because of async
* event or other error, and have egress error
* then drop
*/
if (CQE_TYPE(hw_cqe) == 1)
goto next_cqe;
/* drop peer2peer RTR reads.
*/
if (CQE_WRID_STAG(hw_cqe) == 1)
goto next_cqe;
/*
* Eat completions for unsignaled read WRs.
*/
if (!qhp->wq.sq.oldest_read->signaled) {
advance_oldest_read(&qhp->wq);
goto next_cqe;
}
/*
* Don't write to the HWCQ, create a new read req CQE
* in local memory and move it into the swcq.
*/
create_read_req_cqe(&qhp->wq, hw_cqe, &read_cqe);
hw_cqe = &read_cqe;
advance_oldest_read(&qhp->wq);
}
/* if its a SQ completion, then do the magic to move all the
* unsignaled and now in-order completions into the swcq.
*/
if (SQ_TYPE(hw_cqe)) {
swsqe = &qhp->wq.sq.sw_sq[CQE_WRID_SQ_IDX(hw_cqe)];
swsqe->cqe = *hw_cqe;
swsqe->complete = 1;
flush_completed_wrs(&qhp->wq, &chp->cq);
} else {
swcqe = &chp->cq.sw_queue[chp->cq.sw_pidx];
*swcqe = *hw_cqe;
swcqe->header |= cpu_to_be32(V_CQE_SWCQE(1));
t4_swcq_produce(&chp->cq);
}
next_cqe:
t4_hwcq_consume(&chp->cq);
ret = t4_next_hw_cqe(&chp->cq, &hw_cqe);
}
}
@ -301,25 +451,6 @@ static int cqe_completes_wr(struct t4_cqe *cqe, struct t4_wq *wq)
return 1;
}
void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
{
struct t4_cqe *cqe;
u32 ptr;
*count = 0;
ptr = cq->sw_cidx;
while (ptr != cq->sw_pidx) {
cqe = &cq->sw_queue[ptr];
if ((SQ_TYPE(cqe) || ((CQE_OPCODE(cqe) == FW_RI_READ_RESP) &&
wq->sq.oldest_read)) &&
(CQE_QPID(cqe) == wq->sq.qid))
(*count)++;
if (++ptr == cq->size)
ptr = 0;
}
CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count);
}
void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
{
struct t4_cqe *cqe;
@ -339,71 +470,6 @@ void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count)
CTR3(KTR_IW_CXGBE, "%s cq %p count %d", __func__, cq, *count);
}
static void flush_completed_wrs(struct t4_wq *wq, struct t4_cq *cq)
{
struct t4_swsqe *swsqe;
u16 ptr = wq->sq.cidx;
int count = wq->sq.in_use;
int unsignaled = 0;
swsqe = &wq->sq.sw_sq[ptr];
while (count--)
if (!swsqe->signaled) {
if (++ptr == wq->sq.size)
ptr = 0;
swsqe = &wq->sq.sw_sq[ptr];
unsignaled++;
} else if (swsqe->complete) {
/*
* Insert this completed cqe into the swcq.
*/
CTR3(KTR_IW_CXGBE,
"%s moving cqe into swcq sq idx %u cq idx %u",
__func__, ptr, cq->sw_pidx);
swsqe->cqe.header |= htonl(V_CQE_SWCQE(1));
cq->sw_queue[cq->sw_pidx] = swsqe->cqe;
t4_swcq_produce(cq);
swsqe->signaled = 0;
wq->sq.in_use -= unsignaled;
break;
} else
break;
}
static void create_read_req_cqe(struct t4_wq *wq, struct t4_cqe *hw_cqe,
struct t4_cqe *read_cqe)
{
read_cqe->u.scqe.cidx = wq->sq.oldest_read->idx;
read_cqe->len = cpu_to_be32(wq->sq.oldest_read->read_len);
read_cqe->header = htonl(V_CQE_QPID(CQE_QPID(hw_cqe)) |
V_CQE_SWCQE(SW_CQE(hw_cqe)) |
V_CQE_OPCODE(FW_RI_READ_REQ) |
V_CQE_TYPE(1));
read_cqe->bits_type_ts = hw_cqe->bits_type_ts;
}
/*
* Return a ptr to the next read wr in the SWSQ or NULL.
*/
static void advance_oldest_read(struct t4_wq *wq)
{
u32 rptr = wq->sq.oldest_read - wq->sq.sw_sq + 1;
if (rptr == wq->sq.size)
rptr = 0;
while (rptr != wq->sq.pidx) {
wq->sq.oldest_read = &wq->sq.sw_sq[rptr];
if (wq->sq.oldest_read->opcode == FW_RI_READ_REQ)
return;
if (++rptr == wq->sq.size)
rptr = 0;
}
wq->sq.oldest_read = NULL;
}
/*
* poll_cq
*
@ -449,6 +515,22 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
goto skip_cqe;
}
/*
* skip hw cqe's if the wq is flushed.
*/
if (wq->flushed && !SW_CQE(hw_cqe)) {
ret = -EAGAIN;
goto skip_cqe;
}
/*
* skip TERMINATE cqes...
*/
if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
ret = -EAGAIN;
goto skip_cqe;
}
/*
* Special cqe for drain WR completions...
*/
@ -467,18 +549,37 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
*/
if (RQ_TYPE(hw_cqe) && (CQE_OPCODE(hw_cqe) == FW_RI_READ_RESP)) {
/*
* If this is an unsolicited read response, then the read
/* If we have reached here because of async
* event or other error, and have egress error
* then drop
*/
if (CQE_TYPE(hw_cqe) == 1) {
if (CQE_STATUS(hw_cqe))
t4_set_wq_in_error(wq);
ret = -EAGAIN;
goto skip_cqe;
}
/* If this is an unsolicited read response, then the read
* was generated by the kernel driver as part of peer-2-peer
* connection setup. So ignore the completion.
*/
if (!wq->sq.oldest_read) {
if (CQE_WRID_STAG(hw_cqe) == 1) {
if (CQE_STATUS(hw_cqe))
t4_set_wq_in_error(wq);
ret = -EAGAIN;
goto skip_cqe;
}
/*
* Eat completions for unsignaled read WRs.
*/
if (!wq->sq.oldest_read->signaled) {
advance_oldest_read(wq);
ret = -EAGAIN;
goto skip_cqe;
}
/*
* Don't write to the HWCQ, so create a new read req CQE
* in local memory.
@ -489,14 +590,8 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
}
if (CQE_STATUS(hw_cqe) || t4_wq_in_error(wq)) {
*cqe_flushed = t4_wq_in_error(wq);
*cqe_flushed = (CQE_STATUS(hw_cqe) == T4_ERR_SWFLUSH);
t4_set_wq_in_error(wq);
goto proc_cqe;
}
if (CQE_OPCODE(hw_cqe) == FW_RI_TERMINATE) {
ret = -EAGAIN;
goto skip_cqe;
}
/*
@ -556,9 +651,26 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
* completion.
*/
if (SQ_TYPE(hw_cqe)) {
wq->sq.cidx = CQE_WRID_SQ_IDX(hw_cqe);
CTR2(KTR_IW_CXGBE, "%s completing sq idx %u",
__func__, wq->sq.cidx);
int idx = CQE_WRID_SQ_IDX(hw_cqe);
BUG_ON(idx >= wq->sq.size);
/*
* Account for any unsignaled completions completed by
* this signaled completion. In this case, cidx points
* to the first unsignaled one, and idx points to the
* signaled one. So adjust in_use based on this delta.
* if this is not completing any unsigned wrs, then the
* delta will be 0. Handle wrapping also!
*/
if (idx < wq->sq.cidx)
wq->sq.in_use -= wq->sq.size + idx - wq->sq.cidx;
else
wq->sq.in_use -= idx - wq->sq.cidx;
BUG_ON(wq->sq.in_use <= 0 && wq->sq.in_use >= wq->sq.size);
wq->sq.cidx = (uint16_t)idx;
CTR2(KTR_IW_CXGBE, "%s completing sq idx %u\n",
__func__, wq->sq.cidx);
*cookie = wq->sq.sw_sq[wq->sq.cidx].wr_id;
t4_sq_consume(wq);
} else {
@ -567,6 +679,7 @@ static int poll_cq(struct t4_wq *wq, struct t4_cq *cq, struct t4_cqe *cqe,
*cookie = wq->rq.sw_rq[wq->rq.cidx].wr_id;
BUG_ON(t4_rq_empty(wq));
t4_rq_consume(wq);
goto skip_cqe;
}
flush_wq:
@ -645,6 +758,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
CQE_OPCODE(&cqe) == FW_RI_SEND_WITH_SE_INV) {
wc->ex.invalidate_rkey = CQE_WRID_STAG(&cqe);
wc->wc_flags |= IB_WC_WITH_INVALIDATE;
c4iw_invalidate_mr(qhp->rhp, wc->ex.invalidate_rkey);
}
} else {
switch (CQE_OPCODE(&cqe)) {
@ -664,15 +778,16 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc)
case FW_RI_SEND_WITH_SE:
wc->opcode = IB_WC_SEND;
break;
case FW_RI_BIND_MW:
wc->opcode = IB_WC_BIND_MW;
break;
case FW_RI_LOCAL_INV:
wc->opcode = IB_WC_LOCAL_INV;
break;
case FW_RI_FAST_REGISTER:
wc->opcode = IB_WC_FAST_REG_MR;
wc->opcode = IB_WC_REG_MR;
/* Invalidate the MR if the fastreg failed */
if (CQE_STATUS(&cqe) != T4_ERR_SUCCESS)
c4iw_invalidate_mr(qhp->rhp,
CQE_WRID_FR_STAG(&cqe));
break;
case C4IW_DRAIN_OPCODE:
wc->opcode = IB_WC_SEND;
@ -787,9 +902,11 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq)
}
struct ib_cq *
c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context, struct ib_udata *udata)
{
int entries = attr->cqe;
int vector = attr->comp_vector;
struct c4iw_dev *rhp;
struct c4iw_cq *chp;
struct c4iw_create_cq_resp uresp;
@ -797,9 +914,10 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
int ret;
size_t memsize, hwentries;
struct c4iw_mm_entry *mm, *mm2;
int entries = attr->cqe;
CTR3(KTR_IW_CXGBE, "%s ib_dev %p entries %d", __func__, ibdev, entries);
if (attr->flags)
return ERR_PTR(-EINVAL);
rhp = to_c4iw_dev(ibdev);
@ -807,6 +925,7 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
if (!chp)
return ERR_PTR(-ENOMEM);
if (ib_context)
ucontext = to_c4iw_ucontext(ib_context);
@ -822,9 +941,9 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
entries = roundup(entries, 16);
/*
* Make actual HW queue 2x to avoid cidx_inc overflows.
* Make actual HW queue 2x to avoid cdix_inc overflows.
*/
hwentries = entries * 2;
hwentries = min(entries * 2, rhp->rdev.hw_queue.t4_max_iq_size);
/*
* Make HW queue at least 64 entries so GTS updates aren't too
@ -838,16 +957,11 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
/*
* memsize must be a multiple of the page size if its a user cq.
*/
if (ucontext) {
if (ucontext)
memsize = roundup(memsize, PAGE_SIZE);
hwentries = memsize / sizeof *chp->cq.queue;
while (hwentries > T4_MAX_IQ_SIZE) {
memsize -= PAGE_SIZE;
hwentries = memsize / sizeof *chp->cq.queue;
}
}
chp->cq.size = hwentries;
chp->cq.memsize = memsize;
chp->cq.vector = vector;
ret = create_cq(&rhp->rdev, &chp->cq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
@ -866,6 +980,7 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
goto err2;
if (ucontext) {
ret = -ENOMEM;
mm = kmalloc(sizeof *mm, GFP_KERNEL);
if (!mm)
goto err3;
@ -895,7 +1010,7 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
insert_mmap(ucontext, mm);
mm2->key = uresp.gts_key;
mm2->addr = chp->cq.ugts;
mm2->addr = chp->cq.bar2_pa;
mm2->len = PAGE_SIZE;
insert_mmap(ucontext, mm2);
}
@ -926,16 +1041,16 @@ int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
{
struct c4iw_cq *chp;
int ret;
int ret = 0;
unsigned long flag;
chp = to_c4iw_cq(ibcq);
spin_lock_irqsave(&chp->lock, flag);
ret = t4_arm_cq(&chp->cq,
(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
t4_arm_cq(&chp->cq,
(flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED);
if (flags & IB_CQ_REPORT_MISSED_EVENTS)
ret = t4_cq_notempty(&chp->cq);
spin_unlock_irqrestore(&chp->lock, flag);
if (ret && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
ret = 0;
return ret;
}
#endif

112
sys/dev/cxgbe/iw_cxgbe/device.c
View File

@ -66,7 +66,7 @@ c4iw_release_dev_ucontext(struct c4iw_rdev *rdev,
kfree(entry);
}
list_for_each_safe(pos, nxt, &uctx->qpids) {
list_for_each_safe(pos, nxt, &uctx->cqids) {
entry = list_entry(pos, struct c4iw_qid_list, entry);
list_del_init(&entry->entry);
kfree(entry);
@ -89,22 +89,54 @@ c4iw_rdev_open(struct c4iw_rdev *rdev)
struct adapter *sc = rdev->adap;
struct sge_params *sp = &sc->params.sge;
int rc;
unsigned short ucq_density = 1 << sp->iq_s_qpp; /* # of user CQs/page */
unsigned short udb_density = 1 << sp->eq_s_qpp; /* # of user DB/page */
c4iw_init_dev_ucontext(rdev, &rdev->uctx);
/* XXX: we can probably make this work */
if (sp->eq_s_qpp > PAGE_SHIFT || sp->iq_s_qpp > PAGE_SHIFT) {
device_printf(sc->dev,
"doorbell density too high (eq %d, iq %d, pg %d).\n",
sp->eq_s_qpp, sp->eq_s_qpp, PAGE_SHIFT);
/*
* This implementation assumes udb_density == ucq_density! Eventually
* we might need to support this but for now fail the open. Also the
* cqid and qpid range must match for now.
*/
if (udb_density != ucq_density) {
device_printf(sc->dev, "unsupported udb/ucq densities %u/%u\n",
udb_density, ucq_density);
rc = -EINVAL;
goto err1;
}
if (sc->vres.qp.start != sc->vres.cq.start ||
sc->vres.qp.size != sc->vres.cq.size) {
device_printf(sc->dev, "%s: unsupported qp and cq id ranges "
"qp start %u size %u cq start %u size %u\n", __func__,
sc->vres.qp.start, sc->vres.qp.size, sc->vres.cq.start,
sc->vres.cq.size);
rc = -EINVAL;
goto err1;
}
rdev->qpshift = PAGE_SHIFT - sp->eq_s_qpp;
rdev->qpmask = (1 << sp->eq_s_qpp) - 1;
rdev->qpmask = udb_density - 1;
rdev->cqshift = PAGE_SHIFT - sp->iq_s_qpp;
rdev->cqmask = (1 << sp->iq_s_qpp) - 1;
rdev->cqmask = ucq_density - 1;
CTR5(KTR_IW_CXGBE, "%s dev %s stag start 0x%0x size 0x%0x num stags %d",
__func__, device_get_nameunit(sc->dev), sc->vres.stag.start,
sc->vres.stag.size, c4iw_num_stags(rdev));
CTR5(KTR_IW_CXGBE, "%s pbl start 0x%0x size 0x%0x"
" rq start 0x%0x size 0x%0x", __func__,
sc->vres.pbl.start, sc->vres.pbl.size,
sc->vres.rq.start, sc->vres.rq.size);
CTR5(KTR_IW_CXGBE, "%s:qp qid start %u size %u cq qid start %u size %u",
__func__, sc->vres.qp.start, sc->vres.qp.size,
sc->vres.cq.start, sc->vres.cq.size);
/*TODO
CTR5(KTR_IW_CXGBE, "%s udb %pR db_reg %p gts_reg %p"
"qpmask 0x%x cqmask 0x%x", __func__,
db_reg,gts_reg,rdev->qpmask, rdev->cqmask);
*/
if (c4iw_num_stags(rdev) == 0) {
rc = -EINVAL;
@ -132,8 +164,34 @@ c4iw_rdev_open(struct c4iw_rdev *rdev)
device_printf(sc->dev, "error %d initializing rqt pool\n", rc);
goto err3;
}
rdev->status_page = (struct t4_dev_status_page *)
__get_free_page(GFP_KERNEL);
if (!rdev->status_page) {
rc = -ENOMEM;
goto err4;
}
rdev->status_page->qp_start = sc->vres.qp.start;
rdev->status_page->qp_size = sc->vres.qp.size;
rdev->status_page->cq_start = sc->vres.cq.start;
rdev->status_page->cq_size = sc->vres.cq.size;
/* T5 and above devices don't need Doorbell recovery logic,
* so db_off is always set to '0'.
*/
rdev->status_page->db_off = 0;
rdev->status_page->wc_supported = rdev->adap->iwt.wc_en;
rdev->free_workq = create_singlethread_workqueue("iw_cxgb4_free");
if (!rdev->free_workq) {
rc = -ENOMEM;
goto err5;
}
return (0);
err5:
free_page((unsigned long)rdev->status_page);
err4:
c4iw_rqtpool_destroy(rdev);
err3:
c4iw_pblpool_destroy(rdev);
err2:
@ -144,6 +202,7 @@ c4iw_rdev_open(struct c4iw_rdev *rdev)
static void c4iw_rdev_close(struct c4iw_rdev *rdev)
{
free_page((unsigned long)rdev->status_page);
c4iw_pblpool_destroy(rdev);
c4iw_rqtpool_destroy(rdev);
c4iw_destroy_resource(&rdev->resource);
@ -173,6 +232,34 @@ c4iw_alloc(struct adapter *sc)
}
iwsc->rdev.adap = sc;
/* init various hw-queue params based on lld info */
CTR3(KTR_IW_CXGBE, "%s: Ing. padding boundary is %d, "
"egrsstatuspagesize = %d", __func__,
sc->params.sge.pad_boundary,
sc->params.sge.spg_len);
iwsc->rdev.hw_queue.t4_eq_status_entries =
sc->params.sge.spg_len / EQ_ESIZE;
iwsc->rdev.hw_queue.t4_max_eq_size = 65520;
iwsc->rdev.hw_queue.t4_max_iq_size = 65520;
iwsc->rdev.hw_queue.t4_max_rq_size = 8192 -
iwsc->rdev.hw_queue.t4_eq_status_entries - 1;
iwsc->rdev.hw_queue.t4_max_sq_size =
iwsc->rdev.hw_queue.t4_max_eq_size -
iwsc->rdev.hw_queue.t4_eq_status_entries - 1;
iwsc->rdev.hw_queue.t4_max_qp_depth =
iwsc->rdev.hw_queue.t4_max_rq_size;
iwsc->rdev.hw_queue.t4_max_cq_depth =
iwsc->rdev.hw_queue.t4_max_iq_size - 2;
iwsc->rdev.hw_queue.t4_stat_len = iwsc->rdev.adap->params.sge.spg_len;
/* As T5 and above devices support BAR2 kernel doorbells & WC, we map
* all of BAR2, for both User and Kernel Doorbells-GTS.
*/
iwsc->rdev.bar2_kva = (void __iomem *)((u64)iwsc->rdev.adap->udbs_base);
iwsc->rdev.bar2_pa = vtophys(iwsc->rdev.adap->udbs_base);
iwsc->rdev.bar2_len = rman_get_size(iwsc->rdev.adap->udbs_res);
rc = c4iw_rdev_open(&iwsc->rdev);
if (rc != 0) {
device_printf(sc->dev, "Unable to open CXIO rdev (%d)\n", rc);
@ -185,6 +272,7 @@ c4iw_alloc(struct adapter *sc)
idr_init(&iwsc->mmidr);
spin_lock_init(&iwsc->lock);
mutex_init(&iwsc->rdev.stats.lock);
iwsc->avail_ird = iwsc->rdev.adap->params.max_ird_adapter;
return (iwsc);
}
@ -208,6 +296,12 @@ c4iw_activate(struct adapter *sc)
ASSERT_SYNCHRONIZED_OP(sc);
if (is_t4(sc)) {
device_printf(sc->dev, "No iWARP support for T4 devices, "
"please install T5 or above devices.\n");
return (ENOSYS);
}
if (uld_active(sc, ULD_IWARP)) {
KASSERT(0, ("%s: RDMA already eanbled on sc %p", __func__, sc));
return (0);
@ -330,7 +424,7 @@ c4iw_modevent(module_t mod, int cmd, void *arg)
case MOD_LOAD:
rc = c4iw_mod_load();
if (rc == 0)
printf("iw_cxgbe: Chelsio T4/T5/T6 RDMA driver loaded.\n");
printf("iw_cxgbe: Chelsio T5/T6 RDMA driver loaded.\n");
break;
case MOD_UNLOAD:

137
sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h
View File

@ -115,6 +115,7 @@ struct c4iw_dev_ucontext {
enum c4iw_rdev_flags {
T4_FATAL_ERROR = (1<<0),
T4_STATUS_PAGE_DISABLED = (1<<1),
};
struct c4iw_stat {
@ -133,6 +134,17 @@ struct c4iw_stats {
struct c4iw_stat rqt;
};
struct c4iw_hw_queue {
int t4_eq_status_entries;
int t4_max_eq_size;
int t4_max_iq_size;
int t4_max_rq_size;
int t4_max_sq_size;
int t4_max_qp_depth;
int t4_max_cq_depth;
int t4_stat_len;
};
struct c4iw_rdev {
struct adapter *adap;
struct c4iw_resource resource;
@ -145,6 +157,12 @@ struct c4iw_rdev {
vmem_t *pbl_arena;
u32 flags;
struct c4iw_stats stats;
struct c4iw_hw_queue hw_queue;
struct t4_dev_status_page *status_page;
unsigned long bar2_pa;
void __iomem *bar2_kva;
unsigned int bar2_len;
struct workqueue_struct *free_workq;
};
static inline int c4iw_fatal_error(struct c4iw_rdev *rdev)
@ -178,7 +196,7 @@ static inline void c4iw_wake_up(struct c4iw_wr_wait *wr_waitp, int ret)
static inline int
c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
u32 hwtid, u32 qpid, const char *func)
u32 hwtid, u32 qpid, struct socket *so, const char *func)
{
struct adapter *sc = rdev->adap;
unsigned to = C4IW_WR_TO;
@ -193,6 +211,17 @@ c4iw_wait_for_reply(struct c4iw_rdev *rdev, struct c4iw_wr_wait *wr_waitp,
getmicrotime(&t1);
do {
/* If waiting for reply in rdma_init()/rdma_fini() threads, then
* check if there are any connection errors.
*/
if (so && so->so_error) {
wr_waitp->ret = -ECONNRESET;
CTR5(KTR_IW_CXGBE, "%s - Connection ERROR %u for sock %p"
"tid %u qpid %u", func,
so->so_error, so, hwtid, qpid);
break;
}
ret = wait_for_completion_timeout(&wr_waitp->completion, to);
if (!ret) {
getmicrotime(&t2);
@ -233,6 +262,7 @@ struct c4iw_dev {
struct idr mmidr;
spinlock_t lock;
struct dentry *debugfs_root;
u32 avail_ird;
};
static inline struct c4iw_dev *to_c4iw_dev(struct ib_device *ibdev)
@ -313,6 +343,13 @@ static inline void remove_handle_nolock(struct c4iw_dev *rhp,
_remove_handle(rhp, idr, id, 0);
}
extern int c4iw_max_read_depth;
static inline int cur_max_read_depth(struct c4iw_dev *dev)
{
return min(dev->rdev.adap->params.max_ordird_qp, c4iw_max_read_depth);
}
struct c4iw_pd {
struct ib_pd ibpd;
u32 pdid;
@ -348,6 +385,10 @@ struct c4iw_mr {
struct c4iw_dev *rhp;
u64 kva;
struct tpt_attributes attr;
u64 *mpl;
dma_addr_t mpl_addr;
u32 max_mpl_len;
u32 mpl_len;
};
static inline struct c4iw_mr *to_c4iw_mr(struct ib_mr *ibmr)
@ -367,20 +408,6 @@ static inline struct c4iw_mw *to_c4iw_mw(struct ib_mw *ibmw)
return container_of(ibmw, struct c4iw_mw, ibmw);
}
struct c4iw_fr_page_list {
struct ib_fast_reg_page_list ibpl;
DECLARE_PCI_UNMAP_ADDR(mapping);
dma_addr_t dma_addr;
struct c4iw_dev *dev;
int size;
};
static inline struct c4iw_fr_page_list *to_c4iw_fr_page_list(
struct ib_fast_reg_page_list *ibpl)
{
return container_of(ibpl, struct c4iw_fr_page_list, ibpl);
}
struct c4iw_cq {
struct ib_cq ibcq;
struct c4iw_dev *rhp;
@ -432,6 +459,7 @@ struct c4iw_qp_attributes {
u8 ecode;
u16 sq_db_inc;
u16 rq_db_inc;
u8 send_term;
};
struct c4iw_qp {
@ -442,10 +470,12 @@ struct c4iw_qp {
struct t4_wq wq;
spinlock_t lock;
struct mutex mutex;
atomic_t refcnt;
struct kref kref;
wait_queue_head_t wait;
struct timer_list timer;
int sq_sig_all;
struct work_struct free_work;
struct c4iw_ucontext *ucontext;
};
static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp)
@ -459,6 +489,7 @@ struct c4iw_ucontext {
u32 key;
spinlock_t mmap_lock;
struct list_head mmaps;
struct kref kref;
};
static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
@ -466,6 +497,17 @@ static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
return container_of(c, struct c4iw_ucontext, ibucontext);
}
void _c4iw_free_ucontext(struct kref *kref);
static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext)
{
kref_put(&ucontext->kref, _c4iw_free_ucontext);
}
static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext)
{
kref_get(&ucontext->kref);
}
struct c4iw_mm_entry {
struct list_head entry;
u64 addr;
@ -734,7 +776,8 @@ enum c4iw_ep_flags {
RELEASE_RESOURCES = 2,
CLOSE_SENT = 3,
TIMEOUT = 4,
QP_REFERENCED = 5
QP_REFERENCED = 5,
STOP_MPA_TIMER = 7,
};
enum c4iw_ep_history {
@ -776,8 +819,8 @@ struct c4iw_ep_common {
enum c4iw_ep_state state;
struct kref kref;
struct mutex mutex;
struct sockaddr_in local_addr;
struct sockaddr_in remote_addr;
struct sockaddr_storage local_addr;
struct sockaddr_storage remote_addr;
struct c4iw_wr_wait wr_wait;
unsigned long flags;
unsigned long history;
@ -792,11 +835,13 @@ struct c4iw_listen_ep {
struct c4iw_ep_common com;
unsigned int stid;
int backlog;
struct list_head listen_ep_list; /* list of all listener ep's bound
to one port address */
};
struct c4iw_ep {
struct c4iw_ep_common com;
struct c4iw_ep *parent_ep;
struct c4iw_listen_ep *parent_ep;
struct timer_list timer;
unsigned int atid;
u32 hwtid;
@ -852,6 +897,8 @@ typedef int (*c4iw_handler_func)(struct c4iw_dev *dev, struct mbuf *m);
int c4iw_ep_redirect(void *ctx, struct dst_entry *old, struct dst_entry *new,
struct l2t_entry *l2t);
void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qpid,
struct c4iw_dev_ucontext *uctx);
u32 c4iw_get_resource(struct c4iw_id_table *id_table);
void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry);
int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt, u32 nr_pdid);
@ -875,41 +922,30 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
struct ib_send_wr **bad_wr);
int c4iw_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
struct ib_recv_wr **bad_wr);
int c4iw_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
struct ib_mw_bind *mw_bind);
int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog);
void c4iw_destroy_listen_ep(struct iw_cm_id *cm_id);
int c4iw_create_listen(struct iw_cm_id *cm_id, int backlog);
int c4iw_destroy_listen(struct iw_cm_id *cm_id);
int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param);
int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
void c4iw_qp_add_ref(struct ib_qp *qp);
void c4iw_qp_rem_ref(struct ib_qp *qp);
void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *page_list);
struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(
struct ib_device *device,
int page_list_len);
struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth);
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg);
int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
int c4iw_dealloc_mw(struct ib_mw *mw);
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type);
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64
virt, int acc, struct ib_udata *udata, int mr_id);
virt, int acc, struct ib_udata *udata);
struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
struct ib_phys_buf *buffer_list,
int num_phys_buf,
int acc,
u64 *iova_start);
int c4iw_reregister_phys_mem(struct ib_mr *mr,
int mr_rereg_mask,
struct ib_pd *pd,
struct ib_phys_buf *buffer_list,
int num_phys_buf,
int acc, u64 *iova_start);
int c4iw_dereg_mr(struct ib_mr *ib_mr);
void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
int c4iw_destroy_cq(struct ib_cq *ib_cq);
struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context,
struct ib_udata *udata);
struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context,
struct ib_udata *udata);
int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int c4iw_destroy_qp(struct ib_qp *ib_qp);
@ -926,12 +962,13 @@ void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size);
void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size);
int c4iw_ofld_send(struct c4iw_rdev *rdev, struct mbuf *m);
void c4iw_flush_hw_cq(struct t4_cq *cq);
void c4iw_flush_hw_cq(struct c4iw_cq *cq);
void c4iw_count_rcqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
void c4iw_count_scqes(struct t4_cq *cq, struct t4_wq *wq, int *count);
int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
int __c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp);
int c4iw_flush_rq(struct t4_wq *wq, struct t4_cq *cq, int count);
int c4iw_flush_sq(struct t4_wq *wq, struct t4_cq *cq, int count);
int c4iw_flush_sq(struct c4iw_qp *qhp);
int c4iw_ev_handler(struct sge_iq *, const struct rsp_ctrl *);
u16 c4iw_rqes_posted(struct c4iw_qp *qhp);
int c4iw_post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe);
@ -942,9 +979,9 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx);
void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
struct c4iw_dev_ucontext *uctx);
void c4iw_ev_dispatch(struct c4iw_dev *dev, struct t4_cqe *err_cqe);
void process_newconn(struct iw_cm_id *parent_cm_id,
struct socket *child_so);
void __iomem *c4iw_bar2_addrs(struct c4iw_rdev *rdev, unsigned int qid,
enum t4_bar2_qtype qtype,
unsigned int *pbar2_qid, u64 *pbar2_pa);
extern struct cxgb4_client t4c_client;
extern c4iw_handler_func c4iw_handlers[NUM_CPL_CMDS];
extern int c4iw_max_read_depth;

353
sys/dev/cxgbe/iw_cxgbe/mem.c
View File

@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
#include <common/t4_msg.h>
#include "iw_cxgbe.h"
int use_dsgl = 1;
#define T4_ULPTX_MIN_IO 32
#define C4IW_MAX_INLINE_SIZE 96
@ -50,9 +51,7 @@ static int
mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
{
return ((is_t4(dev->rdev.adap) ||
is_t5(dev->rdev.adap)) &&
length >= 8*1024*1024*1024ULL);
return (is_t5(dev->rdev.adap) && length >= 8*1024*1024*1024ULL);
}
static int
@ -68,10 +67,8 @@ write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
u32 cmd;
cmd = cpu_to_be32(V_ULPTX_CMD(ULP_TX_MEM_WRITE));
if (is_t4(sc))
cmd |= cpu_to_be32(F_ULP_MEMIO_ORDER);
else
cmd |= cpu_to_be32(F_T5_ULP_MEMIO_IMM);
cmd |= cpu_to_be32(F_T5_ULP_MEMIO_IMM);
addr &= 0x7FFFFFF;
CTR3(KTR_IW_CXGBE, "%s addr 0x%x len %u", __func__, addr, len);
@ -124,7 +121,7 @@ write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
len -= C4IW_MAX_INLINE_SIZE;
}
ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, NULL, __func__);
return ret;
}
@ -277,32 +274,6 @@ static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
return ret;
}
static int reregister_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
struct c4iw_mr *mhp, int shift, int npages)
{
u32 stag;
int ret;
if (npages > mhp->attr.pbl_size)
return -ENOMEM;
stag = mhp->attr.stag;
ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid,
FW_RI_STAG_NSMR, mhp->attr.perms,
mhp->attr.mw_bind_enable, mhp->attr.zbva,
mhp->attr.va_fbo, mhp->attr.len, shift - 12,
mhp->attr.pbl_size, mhp->attr.pbl_addr);
if (ret)
return ret;
ret = finish_mem_reg(mhp, stag);
if (ret)
dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
mhp->attr.pbl_addr);
return ret;
}
static int alloc_pbl(struct c4iw_mr *mhp, int npages)
{
mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev,
@ -316,223 +287,6 @@ static int alloc_pbl(struct c4iw_mr *mhp, int npages)
return 0;
}
static int build_phys_page_list(struct ib_phys_buf *buffer_list,
int num_phys_buf, u64 *iova_start,
u64 *total_size, int *npages,
int *shift, __be64 **page_list)
{
u64 mask;
int i, j, n;
mask = 0;
*total_size = 0;
for (i = 0; i < num_phys_buf; ++i) {
if (i != 0 && buffer_list[i].addr & ~PAGE_MASK)
return -EINVAL;
if (i != 0 && i != num_phys_buf - 1 &&
(buffer_list[i].size & ~PAGE_MASK))
return -EINVAL;
*total_size += buffer_list[i].size;
if (i > 0)
mask |= buffer_list[i].addr;
else
mask |= buffer_list[i].addr & PAGE_MASK;
if (i != num_phys_buf - 1)
mask |= buffer_list[i].addr + buffer_list[i].size;
else
mask |= (buffer_list[i].addr + buffer_list[i].size +
PAGE_SIZE - 1) & PAGE_MASK;
}
/* Find largest page shift we can use to cover buffers */
for (*shift = PAGE_SHIFT; *shift < PAGE_SHIFT + M_FW_RI_TPTE_PS;
++(*shift))
if ((1ULL << *shift) & mask)
break;
buffer_list[0].size += buffer_list[0].addr & ((1ULL << *shift) - 1);
buffer_list[0].addr &= ~0ull << *shift;
*npages = 0;
for (i = 0; i < num_phys_buf; ++i)
*npages += (buffer_list[i].size +
(1ULL << *shift) - 1) >> *shift;
if (!*npages)
return -EINVAL;
*page_list = kmalloc(sizeof(u64) * *npages, GFP_KERNEL);
if (!*page_list)
return -ENOMEM;
n = 0;
for (i = 0; i < num_phys_buf; ++i)
for (j = 0;
j < (buffer_list[i].size + (1ULL << *shift) - 1) >> *shift;
++j)
(*page_list)[n++] = cpu_to_be64(buffer_list[i].addr +
((u64) j << *shift));
CTR6(KTR_IW_CXGBE,
"%s va 0x%llx mask 0x%llx shift %d len %lld pbl_size %d", __func__,
(unsigned long long)*iova_start, (unsigned long long)mask, *shift,
(unsigned long long)*total_size, *npages);
return 0;
}
int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask,
struct ib_pd *pd, struct ib_phys_buf *buffer_list,
int num_phys_buf, int acc, u64 *iova_start)
{
struct c4iw_mr mh, *mhp;
struct c4iw_pd *php;
struct c4iw_dev *rhp;
__be64 *page_list = NULL;
int shift = 0;
u64 total_size = 0;
int npages = 0;
int ret;
CTR3(KTR_IW_CXGBE, "%s ib_mr %p ib_pd %p", __func__, mr, pd);
/* There can be no memory windows */
if (atomic_read(&mr->usecnt))
return -EINVAL;
mhp = to_c4iw_mr(mr);
rhp = mhp->rhp;
php = to_c4iw_pd(mr->pd);
/* make sure we are on the same adapter */
if (rhp != php->rhp)
return -EINVAL;
memcpy(&mh, mhp, sizeof *mhp);
if (mr_rereg_mask & IB_MR_REREG_PD)
php = to_c4iw_pd(pd);
if (mr_rereg_mask & IB_MR_REREG_ACCESS) {
mh.attr.perms = c4iw_ib_to_tpt_access(acc);
mh.attr.mw_bind_enable = (acc & IB_ACCESS_MW_BIND) ==
IB_ACCESS_MW_BIND;
}
if (mr_rereg_mask & IB_MR_REREG_TRANS) {
ret = build_phys_page_list(buffer_list, num_phys_buf,
iova_start,
&total_size, &npages,
&shift, &page_list);
if (ret)
return ret;
}
if (mr_exceeds_hw_limits(rhp, total_size)) {
kfree(page_list);
return -EINVAL;
}
ret = reregister_mem(rhp, php, &mh, shift, npages);
kfree(page_list);
if (ret)
return ret;
if (mr_rereg_mask & IB_MR_REREG_PD)
mhp->attr.pdid = php->pdid;
if (mr_rereg_mask & IB_MR_REREG_ACCESS)
mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
if (mr_rereg_mask & IB_MR_REREG_TRANS) {
mhp->attr.zbva = 0;
mhp->attr.va_fbo = *iova_start;
mhp->attr.page_size = shift - 12;
mhp->attr.len = total_size;
mhp->attr.pbl_size = npages;
}
return 0;
}
struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd,
struct ib_phys_buf *buffer_list,
int num_phys_buf, int acc, u64 *iova_start)
{
__be64 *page_list;
int shift;
u64 total_size;
int npages;
struct c4iw_dev *rhp;
struct c4iw_pd *php;
struct c4iw_mr *mhp;
int ret;
CTR2(KTR_IW_CXGBE, "%s ib_pd %p", __func__, pd);
php = to_c4iw_pd(pd);
rhp = php->rhp;
mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
if (!mhp)
return ERR_PTR(-ENOMEM);
mhp->rhp = rhp;
/* First check that we have enough alignment */
if ((*iova_start & ~PAGE_MASK) != (buffer_list[0].addr & ~PAGE_MASK)) {
ret = -EINVAL;
goto err;
}
if (num_phys_buf > 1 &&
((buffer_list[0].addr + buffer_list[0].size) & ~PAGE_MASK)) {
ret = -EINVAL;
goto err;
}
ret = build_phys_page_list(buffer_list, num_phys_buf, iova_start,
&total_size, &npages, &shift,
&page_list);
if (ret)
goto err;
if (mr_exceeds_hw_limits(rhp, total_size)) {
kfree(page_list);
ret = -EINVAL;
goto err;
}
ret = alloc_pbl(mhp, npages);
if (ret) {
kfree(page_list);
goto err;
}
ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr,
npages);
kfree(page_list);
if (ret)
goto err_pbl;
mhp->attr.pdid = php->pdid;
mhp->attr.zbva = 0;
mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
mhp->attr.va_fbo = *iova_start;
mhp->attr.page_size = shift - 12;
mhp->attr.len = total_size;
mhp->attr.pbl_size = npages;
ret = register_mem(rhp, php, mhp, shift);
if (ret)
goto err_pbl;
return &mhp->ibmr;
err_pbl:
c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
mhp->attr.pbl_size << 3);
err:
kfree(mhp);
return ERR_PTR(ret);
}
struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
{
struct c4iw_dev *rhp;
@ -556,12 +310,12 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
mhp->attr.zbva = 0;
mhp->attr.va_fbo = 0;
mhp->attr.page_size = 0;
mhp->attr.len = ~0UL;
mhp->attr.len = ~0ULL;
mhp->attr.pbl_size = 0;
ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid,
FW_RI_STAG_NSMR, mhp->attr.perms,
mhp->attr.mw_bind_enable, 0, 0, ~0UL, 0, 0, 0);
mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0);
if (ret)
goto err1;
@ -578,7 +332,7 @@ struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
}
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt, int acc, struct ib_udata *udata, int mr_id)
u64 virt, int acc, struct ib_udata *udata)
{
__be64 *pages;
int shift, n, len;
@ -680,7 +434,8 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
return ERR_PTR(err);
}
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{
struct c4iw_dev *rhp;
struct c4iw_pd *php;
@ -689,6 +444,9 @@ struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type)
u32 stag = 0;
int ret;
if (type != IB_MW_TYPE_1)
return ERR_PTR(-EINVAL);
php = to_c4iw_pd(pd);
rhp = php->rhp;
mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
@ -732,7 +490,9 @@ int c4iw_dealloc_mw(struct ib_mw *mw)
return 0;
}
struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg)
{
struct c4iw_dev *rhp;
struct c4iw_pd *php;
@ -740,28 +500,43 @@ struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
u32 mmid;
u32 stag = 0;
int ret = 0;
int length = roundup(max_num_sg * sizeof(u64), 32);
php = to_c4iw_pd(pd);
rhp = php->rhp;
if (mr_type != IB_MR_TYPE_MEM_REG ||
max_num_sg > t4_max_fr_depth(
rhp->rdev.adap->params.ulptx_memwrite_dsgl && use_dsgl))
return ERR_PTR(-EINVAL);
mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
if (!mhp) {
ret = -ENOMEM;
goto err;
}
mhp->mpl = dma_alloc_coherent(rhp->ibdev.dma_device,
length, &mhp->mpl_addr, GFP_KERNEL);
if (!mhp->mpl) {
ret = -ENOMEM;
goto err_mpl;
}
mhp->max_mpl_len = length;
mhp->rhp = rhp;
ret = alloc_pbl(mhp, pbl_depth);
ret = alloc_pbl(mhp, max_num_sg);
if (ret)
goto err1;
mhp->attr.pbl_size = pbl_depth;
mhp->attr.pbl_size = max_num_sg;
ret = allocate_stag(&rhp->rdev, &stag, php->pdid,
mhp->attr.pbl_size, mhp->attr.pbl_addr);
mhp->attr.pbl_size, mhp->attr.pbl_addr);
if (ret)
goto err2;
mhp->attr.pdid = php->pdid;
mhp->attr.type = FW_RI_STAG_NSMR;
mhp->attr.stag = stag;
mhp->attr.state = 1;
mhp->attr.state = 0;
mmid = (stag) >> 8;
mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
@ -769,8 +544,7 @@ struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
goto err3;
}
CTR4(KTR_IW_CXGBE, "%s mmid 0x%x mhp %p stag 0x%x", __func__, mmid, mhp,
stag);
PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
return &(mhp->ibmr);
err3:
dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
@ -779,42 +553,36 @@ struct ib_mr *c4iw_alloc_fast_reg_mr(struct ib_pd *pd, int pbl_depth)
c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
mhp->attr.pbl_size << 3);
err1:
dma_free_coherent(rhp->ibdev.dma_device,
mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
err_mpl:
kfree(mhp);
err:
return ERR_PTR(ret);
}
struct ib_fast_reg_page_list *c4iw_alloc_fastreg_pbl(struct ib_device *device,
int page_list_len)
static int c4iw_set_page(struct ib_mr *ibmr, u64 addr)
{
struct c4iw_fr_page_list *c4pl;
struct c4iw_dev *dev = to_c4iw_dev(device);
bus_addr_t dma_addr;
int size = sizeof *c4pl + page_list_len * sizeof(u64);
struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
c4pl = contigmalloc(size,
M_DEVBUF, M_NOWAIT, 0ul, ~0ul, 4096, 0);
if (c4pl)
dma_addr = vtophys(c4pl);
else
return ERR_PTR(-ENOMEM);
if (unlikely(mhp->mpl_len == mhp->max_mpl_len))
return -ENOMEM;
pci_unmap_addr_set(c4pl, mapping, dma_addr);
c4pl->dma_addr = dma_addr;
c4pl->dev = dev;
c4pl->size = size;
c4pl->ibpl.page_list = (u64 *)(c4pl + 1);
c4pl->ibpl.max_page_list_len = page_list_len;
mhp->mpl[mhp->mpl_len++] = addr;
return &c4pl->ibpl;
return 0;
}
void c4iw_free_fastreg_pbl(struct ib_fast_reg_page_list *ibpl)
int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset)
{
struct c4iw_fr_page_list *c4pl = to_c4iw_fr_page_list(ibpl);
contigfree(c4pl, c4pl->size, M_DEVBUF);
struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
mhp->mpl_len = 0;
return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, c4iw_set_page);
}
int c4iw_dereg_mr(struct ib_mr *ib_mr)
{
struct c4iw_dev *rhp;
@ -822,9 +590,6 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr)
u32 mmid;
CTR2(KTR_IW_CXGBE, "%s ib_mr %p", __func__, ib_mr);
/* There can be no memory windows */
if (atomic_read(&ib_mr->usecnt))
return -EINVAL;
mhp = to_c4iw_mr(ib_mr);
rhp = mhp->rhp;
@ -843,4 +608,16 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr)
kfree(mhp);
return 0;
}
void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey)
{
struct c4iw_mr *mhp;
unsigned long flags;
spin_lock_irqsave(&rhp->lock, flags);
mhp = get_mhp(rhp, rkey >> 8);
if (mhp)
mhp->attr.state = 0;
spin_unlock_irqrestore(&rhp->lock, flags);
}
#endif

201
sys/dev/cxgbe/iw_cxgbe/provider.c
View File

@ -44,7 +44,7 @@ __FBSDID("$FreeBSD$");
#include "iw_cxgbe.h"
#include "user.h"
extern int use_dsgl;
static int fastreg_support = 1;
module_param(fastreg_support, int, 0644);
MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default = 1)");
@ -78,24 +78,40 @@ static int c4iw_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
}
static int c4iw_process_mad(struct ib_device *ibdev, int mad_flags,
u8 port_num, struct ib_wc *in_wc,
struct ib_grh *in_grh, struct ib_mad *in_mad,
struct ib_mad *out_mad)
u8 port_num, const struct ib_wc *in_wc,
const struct ib_grh *in_grh,
const struct ib_mad_hdr *in_mad,
size_t in_mad_size,
struct ib_mad_hdr *out_mad,
size_t *out_mad_size,
u16 *out_mad_pkey_index)
{
return -ENOSYS;
}
static int c4iw_dealloc_ucontext(struct ib_ucontext *context)
void _c4iw_free_ucontext(struct kref *kref)
{
struct c4iw_dev *rhp = to_c4iw_dev(context->device);
struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
struct c4iw_ucontext *ucontext;
struct c4iw_dev *rhp;
struct c4iw_mm_entry *mm, *tmp;
CTR2(KTR_IW_CXGBE, "%s context %p", __func__, context);
ucontext = container_of(kref, struct c4iw_ucontext, kref);
rhp = to_c4iw_dev(ucontext->ibucontext.device);
CTR2(KTR_IW_CXGBE, "%s ucontext %p", __func__, ucontext);
list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
kfree(mm);
c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx);
kfree(ucontext);
}
static int c4iw_dealloc_ucontext(struct ib_ucontext *context)
{
struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
CTR2(KTR_IW_CXGBE, "%s context %p", __func__, context);
c4iw_put_ucontext(ucontext);
return 0;
}
@ -104,23 +120,60 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev,
{
struct c4iw_ucontext *context;
struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
static int warned;
struct c4iw_alloc_ucontext_resp uresp;
int ret = 0;
struct c4iw_mm_entry *mm = NULL;
CTR2(KTR_IW_CXGBE, "%s ibdev %p", __func__, ibdev);
PDBG("%s ibdev %p\n", __func__, ibdev);
context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context)
return ERR_PTR(-ENOMEM);
if (!context) {
ret = -ENOMEM;
goto err;
}
c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx);
INIT_LIST_HEAD(&context->mmaps);
spin_lock_init(&context->mmap_lock);
return &context->ibucontext;
}
kref_init(&context->kref);
#ifdef DOT5
static inline pgprot_t t4_pgprot_wc(pgprot_t prot)
{
return pgprot_writecombine(prot);
if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) {
if (!warned++)
log(LOG_ERR, "%s Warning - downlevel libcxgb4 "
"(non-fatal), device status page disabled.\n",
__func__);
rhp->rdev.flags |= T4_STATUS_PAGE_DISABLED;
} else {
mm = kmalloc(sizeof *mm, GFP_KERNEL);
if (!mm)
goto err_free;
uresp.status_page_size = PAGE_SIZE;
spin_lock(&context->mmap_lock);
uresp.status_page_key = context->key;
context->key += PAGE_SIZE;
spin_unlock(&context->mmap_lock);
ret = ib_copy_to_udata(udata, &uresp,
sizeof(uresp) - sizeof(uresp.reserved));
if (ret)
goto err_mm;
mm->key = uresp.status_page_key;
mm->addr = vtophys(rhp->rdev.status_page);
mm->len = PAGE_SIZE;
insert_mmap(context, mm);
}
return &context->ibucontext;
err_mm:
kfree(mm);
err_free:
kfree(context);
err:
return ERR_PTR(ret);
}
#endif
static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
@ -130,12 +183,10 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
int ret = 0;
struct c4iw_mm_entry *mm;
struct c4iw_ucontext *ucontext;
u64 addr, paddr;
u64 addr = 0;
u64 va_regs_res = 0, va_udbs_res = 0;
u64 len_regs_res = 0, len_udbs_res = 0;
CTR3(KTR_IW_CXGBE, "%s:1 ctx %p vma %p", __func__, context, vma);
CTR4(KTR_IW_CXGBE, "%s:1 ctx %p vma %p, vm_start %u", __func__,
context, vma, vma->vm_start);
CTR4(KTR_IW_CXGBE, "%s:1a pgoff 0x%lx key 0x%x len %d", __func__,
vma->vm_pgoff, key, len);
@ -158,59 +209,16 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
addr = mm->addr;
kfree(mm);
va_regs_res = (u64)rman_get_virtual(rdev->adap->regs_res);
len_regs_res = (u64)rman_get_size(rdev->adap->regs_res);
va_udbs_res = (u64)rman_get_virtual(rdev->adap->udbs_res);
len_udbs_res = (u64)rman_get_size(rdev->adap->udbs_res);
/* user DB-GTS registers if addr in udbs_res range,
* else WQ or CQ memory.
* */
if (rdev->adap->iwt.wc_en && addr >= rdev->bar2_pa &&
addr < rdev->bar2_pa + rdev->bar2_len)
vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot);
CTR6(KTR_IW_CXGBE,
"%s:4 addr %p, masync region %p:%p, udb region %p:%p", __func__,
addr, va_regs_res, va_regs_res+len_regs_res, va_udbs_res,
va_udbs_res+len_udbs_res);
if (addr >= va_regs_res && addr < va_regs_res + len_regs_res) {
CTR4(KTR_IW_CXGBE, "%s:5 MA_SYNC addr %p region %p, reglen %u",
__func__, addr, va_regs_res, len_regs_res);
/*
* MA_SYNC register...
*/
paddr = vtophys(addr);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
ret = io_remap_pfn_range(vma, vma->vm_start,
paddr >> PAGE_SHIFT,
len, vma->vm_page_prot);
} else {
if (addr >= va_udbs_res && addr < va_udbs_res + len_udbs_res) {
/*
* Map user DB or OCQP memory...
*/
paddr = vtophys(addr);
CTR4(KTR_IW_CXGBE,
"%s:6 USER DB-GTS addr %p region %p, reglen %u",
__func__, addr, va_udbs_res, len_udbs_res);
#ifdef DOT5
if (!is_t4(rdev->lldi.adapter_type) && map_udb_as_wc)
vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot);
else
#endif
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
ret = io_remap_pfn_range(vma, vma->vm_start,
paddr >> PAGE_SHIFT,
len, vma->vm_page_prot);
} else {
/*
* Map WQ or CQ contig dma memory...
*/
CTR4(KTR_IW_CXGBE,
"%s:7 WQ/CQ addr %p vm_start %u vma %p", __func__,
addr, vma->vm_start, vma);
ret = io_remap_pfn_range(vma, vma->vm_start,
addr >> PAGE_SHIFT,
len, vma->vm_page_prot);
}
}
CTR4(KTR_IW_CXGBE, "%s:8 ctx %p vma %p ret %u", __func__, context, vma,
ret = io_remap_pfn_range(vma, vma->vm_start, addr >> PAGE_SHIFT,
len, vma->vm_page_prot);
CTR4(KTR_IW_CXGBE, "%s:4 ctx %p vma %p ret %u", __func__, context, vma,
ret);
return ret;
}
@ -303,14 +311,17 @@ c4iw_query_gid(struct ib_device *ibdev, u8 port, int index, union ib_gid *gid)
}
static int
c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
struct ib_udata *uhw)
{
struct c4iw_dev *dev = to_c4iw_dev(ibdev);
struct adapter *sc = dev->rdev.adap;
const int spg_ndesc = sc->params.sge.spg_len / EQ_ESIZE;
CTR3(KTR_IW_CXGBE, "%s ibdev %p, props %p", __func__, ibdev, props);
if (uhw->inlen || uhw->outlen)
return -EINVAL;
memset(props, 0, sizeof *props);
memcpy(&props->sys_image_guid, sc->port[0]->vi[0].hw_addr,
ETHER_ADDR_LEN);
@ -322,7 +333,7 @@ c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
props->vendor_part_id = pci_get_device(sc->dev);
props->max_mr_size = T4_MAX_MR_SIZE;
props->max_qp = sc->vres.qp.size / 2;
props->max_qp_wr = T4_MAX_QP_DEPTH(spg_ndesc);
props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth;
props->max_sge = T4_MAX_RECV_SGE;
props->max_sge_rd = 1;
props->max_res_rd_atom = sc->params.max_ird_adapter;
@ -330,11 +341,12 @@ c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
c4iw_max_read_depth);
props->max_qp_init_rd_atom = props->max_qp_rd_atom;
props->max_cq = sc->vres.qp.size;
props->max_cqe = T4_MAX_CQ_DEPTH;
props->max_cqe = dev->rdev.hw_queue.t4_max_cq_depth;
props->max_mr = c4iw_num_stags(&dev->rdev);
props->max_pd = T4_MAX_NUM_PD;
props->local_ca_ack_delay = 0;
props->max_fast_reg_page_list_len = T4_MAX_FR_DEPTH;
props->max_fast_reg_page_list_len =
t4_max_fr_depth(sc->params.ulptx_memwrite_dsgl && use_dsgl);
return (0);
}
@ -405,6 +417,22 @@ static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num,
return 0;
}
static int c4iw_port_immutable(struct ib_device *ibdev, u8 port_num,
struct ib_port_immutable *immutable)
{
struct ib_port_attr attr;
int err;
err = c4iw_query_port(ibdev, port_num, &attr);
if (err)
return err;
immutable->pkey_tbl_len = attr.pkey_tbl_len;
immutable->gid_tbl_len = attr.gid_tbl_len;
immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
return 0;
}
/*
* Returns -errno on error.
@ -472,16 +500,12 @@ c4iw_register_device(struct c4iw_dev *dev)
ibdev->resize_cq = c4iw_resize_cq;
ibdev->poll_cq = c4iw_poll_cq;
ibdev->get_dma_mr = c4iw_get_dma_mr;
ibdev->reg_phys_mr = c4iw_register_phys_mem;
ibdev->rereg_phys_mr = c4iw_reregister_phys_mem;
ibdev->reg_user_mr = c4iw_reg_user_mr;
ibdev->dereg_mr = c4iw_dereg_mr;
ibdev->alloc_mw = c4iw_alloc_mw;
ibdev->bind_mw = c4iw_bind_mw;
ibdev->dealloc_mw = c4iw_dealloc_mw;
ibdev->alloc_fast_reg_mr = c4iw_alloc_fast_reg_mr;
ibdev->alloc_fast_reg_page_list = c4iw_alloc_fastreg_pbl;
ibdev->free_fast_reg_page_list = c4iw_free_fastreg_pbl;
ibdev->alloc_mr = c4iw_alloc_mr;
ibdev->map_mr_sg = c4iw_map_mr_sg;
ibdev->attach_mcast = c4iw_multicast_attach;
ibdev->detach_mcast = c4iw_multicast_detach;
ibdev->process_mad = c4iw_process_mad;
@ -498,9 +522,8 @@ c4iw_register_device(struct c4iw_dev *dev)
iwcm->connect = c4iw_connect;
iwcm->accept = c4iw_accept_cr;
iwcm->reject = c4iw_reject_cr;
iwcm->create_listen_ep = c4iw_create_listen_ep;
iwcm->destroy_listen_ep = c4iw_destroy_listen_ep;
iwcm->newconn = process_newconn;
iwcm->create_listen = c4iw_create_listen;
iwcm->destroy_listen = c4iw_destroy_listen;
iwcm->add_ref = c4iw_qp_add_ref;
iwcm->rem_ref = c4iw_qp_rem_ref;
iwcm->get_qp = c4iw_get_qp;

727
sys/dev/cxgbe/iw_cxgbe/qp.c
View File

File diff suppressed because it is too large Load Diff

209
sys/dev/cxgbe/iw_cxgbe/t4.h
View File

@ -33,6 +33,8 @@
#ifndef __T4_H__
#define __T4_H__
#include "common/t4_regs_values.h"
#include "common/t4_regs.h"
/*
* Fixme: Adding missing defines
*/
@ -60,14 +62,8 @@
#define CIDXINC(x) ((x) << CIDXINC_SHIFT)
#define T4_MAX_NUM_PD 65536
#define T4_MAX_EQ_SIZE 65520
#define T4_MAX_IQ_SIZE 65520
#define T4_MAX_RQ_SIZE(n) (8192 - (n) - 1)
#define T4_MAX_SQ_SIZE(n) (T4_MAX_EQ_SIZE - (n) - 1)
#define T4_MAX_QP_DEPTH(n) (T4_MAX_RQ_SIZE(n))
#define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 2)
#define T4_MAX_MR_SIZE (~0ULL - 1)
#define T4_PAGESIZE_MASK 0xffffffff000 /* 4KB-8TB */
#define T4_MAX_MR_SIZE (~0ULL)
#define T4_PAGESIZE_MASK 0xffffffff000 /* 4KB-8TB */
#define T4_STAG_UNSET 0xffffffff
#define T4_FW_MAJ 0
#define A_PCIE_MA_SYNC 0x30b4
@ -102,7 +98,14 @@ struct t4_status_page {
sizeof(struct fw_ri_isgl)) / sizeof(struct fw_ri_sge))
#define T4_MAX_FR_IMMD ((T4_SQ_NUM_BYTES - sizeof(struct fw_ri_fr_nsmr_wr) - \
sizeof(struct fw_ri_immd)) & ~31UL)
#define T4_MAX_FR_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
#define T4_MAX_FR_IMMD_DEPTH (T4_MAX_FR_IMMD / sizeof(u64))
#define T4_MAX_FR_DSGL 1024
#define T4_MAX_FR_DSGL_DEPTH (T4_MAX_FR_DSGL / sizeof(u64))
static inline int t4_max_fr_depth(int use_dsgl)
{
return use_dsgl ? T4_MAX_FR_DSGL_DEPTH : T4_MAX_FR_IMMD_DEPTH;
}
#define T4_RQ_NUM_SLOTS 2
#define T4_RQ_NUM_BYTES (T4_EQ_ENTRY_SIZE * T4_RQ_NUM_SLOTS)
@ -116,6 +119,7 @@ union t4_wr {
struct fw_ri_rdma_read_wr read;
struct fw_ri_bind_mw_wr bind;
struct fw_ri_fr_nsmr_wr fr;
struct fw_ri_fr_nsmr_tpte_wr fr_tpte;
struct fw_ri_inv_lstag_wr inv;
struct t4_status_page status;
__be64 flits[T4_EQ_ENTRY_SIZE / sizeof(__be64) * T4_SQ_NUM_SLOTS];
@ -191,7 +195,7 @@ struct t4_cqe {
__be32 msn;
} rcqe;
struct {
u32 nada1;
u32 stag;
u16 nada2;
u16 cidx;
} scqe;
@ -254,6 +258,7 @@ struct t4_cqe {
/* used for SQ completion processing */
#define CQE_WRID_SQ_IDX(x) ((x)->u.scqe.cidx)
#define CQE_WRID_FR_STAG(x) (be32_to_cpu((x)->u.scqe.stag))
/* generic accessor macros */
#define CQE_WRID_HI(x) ((x)->u.gen.wrid_hi)
@ -289,24 +294,44 @@ struct t4_swsqe {
int complete;
int signaled;
u16 idx;
int flushed;
struct timespec host_ts;
u64 sge_ts;
};
static inline pgprot_t t4_pgprot_wc(pgprot_t prot)
{
#if defined(__i386__) || defined(__x86_64__) || defined(CONFIG_PPC64)
return pgprot_writecombine(prot);
#else
return pgprot_noncached(prot);
#endif
}
enum {
T4_SQ_ONCHIP = (1<<0),
};
struct t4_sq {
union t4_wr *queue;
bus_addr_t dma_addr;
DECLARE_PCI_UNMAP_ADDR(mapping);
DEFINE_DMA_UNMAP_ADDR(mapping);
unsigned long phys_addr;
struct t4_swsqe *sw_sq;
struct t4_swsqe *oldest_read;
u64 udb;
void __iomem *bar2_va;
u64 bar2_pa;
size_t memsize;
u32 bar2_qid;
u32 qid;
u16 in_use;
u16 size;
u16 cidx;
u16 pidx;
u16 wq_pidx;
u16 wq_pidx_inc;
u16 flags;
short flush_cidx;
};
struct t4_swrqe {
@ -316,10 +341,13 @@ struct t4_swrqe {
struct t4_rq {
union t4_recv_wr *queue;
bus_addr_t dma_addr;
DECLARE_PCI_UNMAP_ADDR(mapping);
DEFINE_DMA_UNMAP_ADDR(mapping);
unsigned long phys_addr;
struct t4_swrqe *sw_rq;
u64 udb;
void __iomem *bar2_va;
u64 bar2_pa;
size_t memsize;
u32 bar2_qid;
u32 qid;
u32 msn;
u32 rqt_hwaddr;
@ -329,14 +357,14 @@ struct t4_rq {
u16 cidx;
u16 pidx;
u16 wq_pidx;
u16 wq_pidx_inc;
};
struct t4_wq {
struct t4_sq sq;
struct t4_rq rq;
void __iomem *db;
void __iomem *gts;
struct c4iw_rdev *rdev;
int flushed;
};
static inline int t4_rqes_posted(struct t4_wq *wq)
@ -384,7 +412,12 @@ static inline u16 t4_rq_host_wq_pidx(struct t4_wq *wq)
static inline u16 t4_rq_wq_size(struct t4_wq *wq)
{
return wq->rq.size * T4_RQ_NUM_SLOTS;
return wq->rq.size * T4_RQ_NUM_SLOTS;
}
static inline int t4_sq_onchip(struct t4_sq *sq)
{
return sq->flags & T4_SQ_ONCHIP;
}
static inline int t4_sq_empty(struct t4_wq *wq)
@ -414,6 +447,9 @@ static inline void t4_sq_produce(struct t4_wq *wq, u8 len16)
static inline void t4_sq_consume(struct t4_wq *wq)
{
BUG_ON(wq->sq.in_use < 1);
if (wq->sq.cidx == wq->sq.flush_cidx)
wq->sq.flush_cidx = -1;
wq->sq.in_use--;
if (++wq->sq.cidx == wq->sq.size)
wq->sq.cidx = 0;
@ -429,16 +465,69 @@ static inline u16 t4_sq_wq_size(struct t4_wq *wq)
return wq->sq.size * T4_SQ_NUM_SLOTS;
}
static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc)
/* This function copies 64 byte coalesced work request to memory
* mapped BAR2 space. For coalesced WRs, the SGE fetches data
* from the FIFO instead of from Host.
*/
static inline void pio_copy(u64 __iomem *dst, u64 *src)
{
wmb();
writel(QID(wq->sq.qid) | PIDX(inc), wq->db);
int count = 8;
while (count) {
writeq(*src, dst);
src++;
dst++;
count--;
}
}
static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc)
static inline void
t4_ring_sq_db(struct t4_wq *wq, u16 inc, union t4_wr *wqe, u8 wc)
{
/* Flush host queue memory writes. */
wmb();
writel(QID(wq->rq.qid) | PIDX(inc), wq->db);
if (wc && inc == 1 && wq->sq.bar2_qid == 0 && wqe) {
CTR2(KTR_IW_CXGBE, "%s: WC wq->sq.pidx = %d\n",
__func__, wq->sq.pidx);
pio_copy((u64 __iomem *)
((u64)wq->sq.bar2_va + SGE_UDB_WCDOORBELL),
(u64 *)wqe);
} else {
CTR2(KTR_IW_CXGBE, "%s: DB wq->sq.pidx = %d\n",
__func__, wq->sq.pidx);
writel(V_PIDX_T5(inc) | V_QID(wq->sq.bar2_qid),
(void __iomem *)((u64)wq->sq.bar2_va +
SGE_UDB_KDOORBELL));
}
/* Flush user doorbell area writes. */
wmb();
return;
}
static inline void
t4_ring_rq_db(struct t4_wq *wq, u16 inc, union t4_recv_wr *wqe, u8 wc)
{
/* Flush host queue memory writes. */
wmb();
if (wc && inc == 1 && wq->rq.bar2_qid == 0 && wqe) {
CTR2(KTR_IW_CXGBE, "%s: WC wq->rq.pidx = %d\n",
__func__, wq->rq.pidx);
pio_copy((u64 __iomem *)((u64)wq->rq.bar2_va +
SGE_UDB_WCDOORBELL), (u64 *)wqe);
} else {
CTR2(KTR_IW_CXGBE, "%s: DB wq->rq.pidx = %d\n",
__func__, wq->rq.pidx);
writel(V_PIDX_T5(inc) | V_QID(wq->rq.bar2_qid),
(void __iomem *)((u64)wq->rq.bar2_va +
SGE_UDB_KDOORBELL));
}
/* Flush user doorbell area writes. */
wmb();
return;
}
static inline int t4_wq_in_error(struct t4_wq *wq)
@ -451,17 +540,24 @@ static inline void t4_set_wq_in_error(struct t4_wq *wq)
wq->rq.queue[wq->rq.size].status.qp_err = 1;
}
enum t4_cq_flags {
CQ_ARMED = 1,
};
struct t4_cq {
struct t4_cqe *queue;
bus_addr_t dma_addr;
DECLARE_PCI_UNMAP_ADDR(mapping);
DEFINE_DMA_UNMAP_ADDR(mapping);
struct t4_cqe *sw_queue;
void __iomem *gts;
void __iomem *bar2_va;
u64 bar2_pa;
u32 bar2_qid;
struct c4iw_rdev *rdev;
u64 ugts;
size_t memsize;
__be64 bits_type_ts;
u32 cqid;
u32 qid_mask;
int vector;
u16 size; /* including status page */
u16 cidx;
u16 sw_pidx;
@ -470,21 +566,34 @@ struct t4_cq {
u16 cidx_inc;
u8 gen;
u8 error;
unsigned long flags;
};
static inline void write_gts(struct t4_cq *cq, u32 val)
{
writel(val | V_INGRESSQID(cq->bar2_qid),
(void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));
}
static inline int t4_clear_cq_armed(struct t4_cq *cq)
{
return test_and_clear_bit(CQ_ARMED, &cq->flags);
}
static inline int t4_arm_cq(struct t4_cq *cq, int se)
{
u32 val;
set_bit(CQ_ARMED, &cq->flags);
while (cq->cidx_inc > CIDXINC_MASK) {
val = SEINTARM(0) | CIDXINC(CIDXINC_MASK) | TIMERREG(7) |
INGRESSQID(cq->cqid);
writel(val, cq->gts);
val = SEINTARM(0) | CIDXINC(CIDXINC_MASK) | TIMERREG(7);
writel(val | V_INGRESSQID(cq->bar2_qid),
(void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));
cq->cidx_inc -= CIDXINC_MASK;
}
val = SEINTARM(se) | CIDXINC(cq->cidx_inc) | TIMERREG(6) |
INGRESSQID(cq->cqid);
writel(val, cq->gts);
val = SEINTARM(se) | CIDXINC(cq->cidx_inc) | TIMERREG(6);
writel(val | V_INGRESSQID(cq->bar2_qid),
(void __iomem *)((u64)cq->bar2_va + SGE_UDB_GTS));
cq->cidx_inc = 0;
return 0;
}
@ -492,12 +601,19 @@ static inline int t4_arm_cq(struct t4_cq *cq, int se)
static inline void t4_swcq_produce(struct t4_cq *cq)
{
cq->sw_in_use++;
if (cq->sw_in_use == cq->size) {
CTR2(KTR_IW_CXGBE, "%s cxgb4 sw cq overflow cqid %u\n",
__func__, cq->cqid);
cq->error = 1;
BUG_ON(1);
}
if (++cq->sw_pidx == cq->size)
cq->sw_pidx = 0;
}
static inline void t4_swcq_consume(struct t4_cq *cq)
{
BUG_ON(cq->sw_in_use < 1);
cq->sw_in_use--;
if (++cq->sw_cidx == cq->size)
cq->sw_cidx = 0;
@ -509,9 +625,8 @@ static inline void t4_hwcq_consume(struct t4_cq *cq)
if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == M_CIDXINC) {
u32 val;
val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7) |
INGRESSQID(cq->cqid);
writel(val, cq->gts);
val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7);
write_gts(cq, val);
cq->cidx_inc = 0;
}
if (++cq->cidx == cq->size) {
@ -525,6 +640,11 @@ static inline int t4_valid_cqe(struct t4_cq *cq, struct t4_cqe *cqe)
return (CQE_GENBIT(cqe) == cq->gen);
}
static inline int t4_cq_notempty(struct t4_cq *cq)
{
return cq->sw_in_use || t4_valid_cqe(cq, &cq->queue[cq->cidx]);
}
static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
{
int ret;
@ -539,7 +659,11 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
ret = -EOVERFLOW;
cq->error = 1;
printk(KERN_ERR MOD "cq overflow cqid %u\n", cq->cqid);
BUG_ON(1);
} else if (t4_valid_cqe(cq, &cq->queue[cq->cidx])) {
/* Ensure CQE is flushed to memory */
rmb();
*cqe = &cq->queue[cq->cidx];
ret = 0;
} else
@ -549,6 +673,13 @@ static inline int t4_next_hw_cqe(struct t4_cq *cq, struct t4_cqe **cqe)
static inline struct t4_cqe *t4_next_sw_cqe(struct t4_cq *cq)
{
if (cq->sw_in_use == cq->size) {
CTR2(KTR_IW_CXGBE, "%s cxgb4 sw cq overflow cqid %u\n",
__func__, cq->cqid);
cq->error = 1;
BUG_ON(1);
return NULL;
}
if (cq->sw_in_use)
return &cq->sw_queue[cq->sw_cidx];
return NULL;
@ -576,4 +707,14 @@ static inline void t4_set_cq_in_error(struct t4_cq *cq)
{
((struct t4_status_page *)&cq->queue[cq->size])->qp_err = 1;
}
struct t4_dev_status_page {
u8 db_off;
u8 wc_supported;
u16 pad2;
u32 pad3;
u64 qp_start;
u64 qp_size;
u64 cq_start;
u64 cq_size;
};
#endif

8
sys/dev/cxgbe/iw_cxgbe/user.h
View File

@ -34,7 +34,7 @@
#ifndef __C4IW_USER_H__
#define __C4IW_USER_H__
#define C4IW_UVERBS_ABI_VERSION 2
#define C4IW_UVERBS_ABI_VERSION 3
/*
* Make sure that all structs defined in this file remain laid out so
@ -68,4 +68,10 @@ struct c4iw_create_qp_resp {
__u32 qid_mask;
__u32 flags;
};
struct c4iw_alloc_ucontext_resp {
__u64 status_page_key;
__u32 status_page_size;
__u32 reserved; /* explicit padding (optional for i386) */
};
#endif

5
sys/dev/cxgbe/offload.h
View File

@ -151,7 +151,10 @@ struct tom_tunables {
int tx_align;
int tx_zcopy;
};
/* iWARP driver tunables */
struct iw_tunables {
int wc_en;
};
#ifdef TCP_OFFLOAD
int t4_register_uld(struct uld_info *);
int t4_unregister_uld(struct uld_info *);

6
sys/dev/cxgbe/t4_main.c
View File

@ -469,7 +469,7 @@ TUNABLE_INT("hw.cxgbe.iscsicaps_allowed", &t4_iscsicaps_allowed);
static int t4_fcoecaps_allowed = 0;
TUNABLE_INT("hw.cxgbe.fcoecaps_allowed", &t4_fcoecaps_allowed);
static int t5_write_combine = 0;
static int t5_write_combine = 1;
TUNABLE_INT("hw.cxl.write_combine", &t5_write_combine);
static int t4_num_vis = 1;
@ -2331,6 +2331,7 @@ t4_map_bar_2(struct adapter *sc)
setbit(&sc->doorbells, DOORBELL_WCWR);
setbit(&sc->doorbells, DOORBELL_UDBWC);
} else {
t5_write_combine = 0;
device_printf(sc->dev,
"couldn't enable write combining: %d\n",
rc);
@ -2340,7 +2341,10 @@ t4_map_bar_2(struct adapter *sc)
t4_write_reg(sc, A_SGE_STAT_CFG,
V_STATSOURCE_T5(7) | mode);
}
#else
t5_write_combine = 0;
#endif
sc->iwt.wc_en = t5_write_combine;
}
return (0);

3
sys/modules/cxgbe/iw_cxgbe/Makefile
View File

@ -24,6 +24,9 @@ SRCS+= resource.c
SRCS+= vnode_if.h
CFLAGS+= -I${CXGBE} -I${SRCTOP}/sys/ofed/include -DLINUX_TYPES_DEFINED
CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include
CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM
CFLAGS+= -DINET6 -DINET
.include <bsd.kmod.mk>