diff --git a/contrib/ofed/libcxgb4/src/cq.c b/contrib/ofed/libcxgb4/src/cq.c index f028a22d9a60..ed5b66b4507e 100644 --- a/contrib/ofed/libcxgb4/src/cq.c +++ b/contrib/ofed/libcxgb4/src/cq.c @@ -697,7 +697,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ibv_wc *wc) default: PDBG("Unexpected cqe_status 0x%x for QPID=0x%0x\n", CQE_STATUS(&cqe), CQE_QPID(&cqe)); - ret = -EINVAL; + wc->status = IBV_WC_FATAL_ERR; } } if (wc->status && wc->status != IBV_WC_WR_FLUSH_ERR) diff --git a/contrib/ofed/libcxgb4/src/dev.c b/contrib/ofed/libcxgb4/src/dev.c index 322fa0c86817..33c10de237e5 100644 --- a/contrib/ofed/libcxgb4/src/dev.c +++ b/contrib/ofed/libcxgb4/src/dev.c @@ -54,7 +54,6 @@ struct { \ unsigned vendor; \ unsigned device; \ - unsigned chip_version; \ } hca_table[] = { #define CH_PCI_DEVICE_ID_FUNCTION \ @@ -64,7 +63,6 @@ { \ .vendor = PCI_VENDOR_ID_CHELSIO, \ .device = (__DeviceID), \ - .chip_version = CHELSIO_PCI_ID_CHIP_VERSION(__DeviceID), \ } #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ @@ -493,7 +491,8 @@ found: } PDBG("%s found vendor %d device %d type %d\n", - __FUNCTION__, vendor, device, hca_table[i].chip_version); + __FUNCTION__, vendor, device, + CHELSIO_PCI_ID_CHIP_VERSION(hca_table[i].device)); dev = calloc(1, sizeof *dev); if (!dev) { @@ -502,7 +501,7 @@ found: pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE); dev->ibv_dev.ops = c4iw_dev_ops; - dev->chip_version = hca_table[i].chip_version; + dev->chip_version = CHELSIO_PCI_ID_CHIP_VERSION(hca_table[i].device); dev->abi_version = abi_version; PDBG("%s device claimed\n", __FUNCTION__); diff --git a/contrib/ofed/libcxgb4/src/libcxgb4.h b/contrib/ofed/libcxgb4/src/libcxgb4.h index 7110aa30864f..30afcf301af2 100644 --- a/contrib/ofed/libcxgb4/src/libcxgb4.h +++ b/contrib/ofed/libcxgb4/src/libcxgb4.h @@ -69,6 +69,11 @@ static inline int dev_is_t5(struct c4iw_dev *dev) return dev->chip_version == CHELSIO_T5; } +static inline int dev_is_t4(struct c4iw_dev *dev) +{ + return dev->chip_version == CHELSIO_T4; +} + struct c4iw_context { struct ibv_context ibv_ctx; struct t4_dev_status_page *status_page; diff --git a/contrib/ofed/libcxgb4/src/qp.c b/contrib/ofed/libcxgb4/src/qp.c index 9a8629138f21..71cc49b6e8ff 100644 --- a/contrib/ofed/libcxgb4/src/qp.c +++ b/contrib/ofed/libcxgb4/src/qp.c @@ -362,7 +362,7 @@ int c4iw_post_send(struct ibv_qp *ibqp, struct ibv_send_wr *wr, err = build_rdma_read(wqe, wr, &len16); if (err) break; - swsqe->read_len = wr->sg_list[0].length; + swsqe->read_len = wr->sg_list ? wr->sg_list[0].length : 0; if (!qhp->wq.sq.oldest_read) qhp->wq.sq.oldest_read = swsqe; break; diff --git a/contrib/ofed/libcxgb4/src/t4.h b/contrib/ofed/libcxgb4/src/t4.h index ce48244ce340..c5120a6a61b3 100644 --- a/contrib/ofed/libcxgb4/src/t4.h +++ b/contrib/ofed/libcxgb4/src/t4.h @@ -328,6 +328,7 @@ struct t4_sq { volatile u32 *udb; size_t memsize; u32 qid; + u32 bar2_qid; void *ma_sync; u16 in_use; u16 size; @@ -336,6 +337,7 @@ struct t4_sq { u16 wq_pidx; u16 flags; short flush_cidx; + int wc_reg_available; }; struct t4_swrqe { @@ -348,6 +350,7 @@ struct t4_rq { volatile u32 *udb; size_t memsize; u32 qid; + u32 bar2_qid; u32 msn; u32 rqt_hwaddr; u16 rqt_size; @@ -356,6 +359,7 @@ struct t4_rq { u16 cidx; u16 pidx; u16 wq_pidx; + int wc_reg_available; }; struct t4_wq { @@ -485,14 +489,14 @@ static inline void t4_ring_sq_db(struct t4_wq *wq, u16 inc, u8 t5, u8 len16, { wc_wmb(); if (t5) { - if (t5_en_wc && inc == 1) { + if (t5_en_wc && inc == 1 && wq->sq.wc_reg_available) { PDBG("%s: WC wq->sq.pidx = %d; len16=%d\n", __func__, wq->sq.pidx, len16); copy_wqe_to_udb(wq->sq.udb + 14, wqe); } else { PDBG("%s: DB wq->sq.pidx = %d; len16=%d\n", __func__, wq->sq.pidx, len16); - writel(V_PIDX_T5(inc), wq->sq.udb); + writel(V_QID(wq->sq.bar2_qid) | V_PIDX_T5(inc), wq->sq.udb); } wc_wmb(); return; @@ -518,14 +522,14 @@ static inline void t4_ring_rq_db(struct t4_wq *wq, u16 inc, u8 t5, u8 len16, { wc_wmb(); if (t5) { - if (t5_en_wc && inc == 1) { + if (t5_en_wc && inc == 1 && wq->sq.wc_reg_available) { PDBG("%s: WC wq->rq.pidx = %d; len16=%d\n", __func__, wq->rq.pidx, len16); copy_wqe_to_udb(wq->rq.udb + 14, wqe); } else { PDBG("%s: DB wq->rq.pidx = %d; len16=%d\n", __func__, wq->rq.pidx, len16); - writel(V_PIDX_T5(inc), wq->rq.udb); + writel(V_QID(wq->rq.bar2_qid) | V_PIDX_T5(inc), wq->rq.udb); } wc_wmb(); return; diff --git a/contrib/ofed/libcxgb4/src/verbs.c b/contrib/ofed/libcxgb4/src/verbs.c index 93d1e499aaaa..4d2a69d27427 100644 --- a/contrib/ofed/libcxgb4/src/verbs.c +++ b/contrib/ofed/libcxgb4/src/verbs.c @@ -213,7 +213,7 @@ struct ibv_cq *c4iw_create_cq(struct ibv_context *context, int cqe, goto err3; if (dev_is_t5(chp->rhp)) - chp->cq.ugts += 3; + chp->cq.ugts += 5; else chp->cq.ugts += 1; chp->cq.sw_queue = calloc(chp->cq.size, sizeof *chp->cq.queue); @@ -460,8 +460,14 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, goto err3; } qhp->wq.sq.udb = dbva; - if (dev_is_t5(qhp->rhp)) { - qhp->wq.sq.udb += (128*(qhp->wq.sq.qid & qhp->wq.qid_mask))/4; + if (!dev_is_t4(qhp->rhp)) { + unsigned long segment_offset = 128 * (qhp->wq.sq.qid & qhp->wq.qid_mask); + + if (segment_offset < c4iw_page_size) { + qhp->wq.sq.udb += segment_offset / 4; + qhp->wq.sq.wc_reg_available = 1; + } else + qhp->wq.sq.bar2_qid = qhp->wq.sq.qid & qhp->wq.qid_mask; qhp->wq.sq.udb += 2; } @@ -479,8 +485,14 @@ static struct ibv_qp *create_qp(struct ibv_pd *pd, if (dbva == MAP_FAILED) goto err5; qhp->wq.rq.udb = dbva; - if (dev_is_t5(qhp->rhp)) { - qhp->wq.rq.udb += (128*(qhp->wq.rq.qid & qhp->wq.qid_mask))/4; + if (!dev_is_t4(qhp->rhp)) { + unsigned long segment_offset = 128 * (qhp->wq.rq.qid & qhp->wq.qid_mask); + + if (segment_offset < c4iw_page_size) { + qhp->wq.rq.udb += segment_offset / 4; + qhp->wq.rq.wc_reg_available = 1; + } else + qhp->wq.rq.bar2_qid = qhp->wq.rq.qid & qhp->wq.qid_mask; qhp->wq.rq.udb += 2; } qhp->wq.rq.queue = mmap(NULL, qhp->wq.rq.memsize, diff --git a/sys/dev/cxgbe/iw_cxgbe/cm.c b/sys/dev/cxgbe/iw_cxgbe/cm.c index c884f5a58f9c..f196b20c3f48 100644 --- a/sys/dev/cxgbe/iw_cxgbe/cm.c +++ b/sys/dev/cxgbe/iw_cxgbe/cm.c @@ -80,7 +80,7 @@ static spinlock_t timeout_lock; static void process_req(struct work_struct *ctx); static void start_ep_timer(struct c4iw_ep *ep); -static void stop_ep_timer(struct c4iw_ep *ep); +static int stop_ep_timer(struct c4iw_ep *ep); static int set_tcpinfo(struct c4iw_ep *ep); static enum c4iw_ep_state state_read(struct c4iw_ep_common *epc); static void __state_set(struct c4iw_ep_common *epc, enum c4iw_ep_state tostate); @@ -96,14 +96,14 @@ static void send_mpa_req(struct c4iw_ep *ep); static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen); static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen); static void close_complete_upcall(struct c4iw_ep *ep, int status); -static int abort_connection(struct c4iw_ep *ep); +static int send_abort(struct c4iw_ep *ep); static void peer_close_upcall(struct c4iw_ep *ep); static void peer_abort_upcall(struct c4iw_ep *ep); static void connect_reply_upcall(struct c4iw_ep *ep, int status); static int connect_request_upcall(struct c4iw_ep *ep); static void established_upcall(struct c4iw_ep *ep); -static void process_mpa_reply(struct c4iw_ep *ep); -static void process_mpa_request(struct c4iw_ep *ep); +static int process_mpa_reply(struct c4iw_ep *ep); +static int process_mpa_request(struct c4iw_ep *ep); static void process_peer_close(struct c4iw_ep *ep); static void process_conn_error(struct c4iw_ep *ep); static void process_close_complete(struct c4iw_ep *ep); @@ -123,11 +123,11 @@ static void release_ep_resources(struct c4iw_ep *ep); } while (0) #define STOP_EP_TIMER(ep) \ - do { \ + ({ \ CTR3(KTR_IW_CXGBE, "stop_ep_timer (%s:%d) ep %p", \ __func__, __LINE__, (ep)); \ stop_ep_timer(ep); \ - } while (0) + }) #ifdef KTR static char *states[] = { @@ -147,6 +147,34 @@ static char *states[] = { }; #endif + +static void deref_cm_id(struct c4iw_ep_common *epc) +{ + epc->cm_id->rem_ref(epc->cm_id); + epc->cm_id = NULL; + set_bit(CM_ID_DEREFED, &epc->history); +} + +static void ref_cm_id(struct c4iw_ep_common *epc) +{ + set_bit(CM_ID_REFED, &epc->history); + epc->cm_id->add_ref(epc->cm_id); +} + +static void deref_qp(struct c4iw_ep *ep) +{ + c4iw_qp_rem_ref(&ep->com.qp->ibqp); + clear_bit(QP_REFERENCED, &ep->com.flags); + set_bit(QP_DEREFED, &ep->com.history); +} + +static void ref_qp(struct c4iw_ep *ep) +{ + set_bit(QP_REFERENCED, &ep->com.flags); + set_bit(QP_REFED, &ep->com.history); + c4iw_qp_add_ref(&ep->com.qp->ibqp); +} + static void process_req(struct work_struct *ctx) { @@ -304,9 +332,7 @@ process_peer_close(struct c4iw_ep *ep) disconnect = 0; STOP_EP_TIMER(ep); close_socket(&ep->com, 0); - ep->com.cm_id->rem_ref(ep->com.cm_id); - ep->com.cm_id = NULL; - ep->com.qp = NULL; + deref_cm_id(&ep->com); release = 1; break; @@ -490,6 +516,7 @@ process_close_complete(struct c4iw_ep *ep) /* The cm_id may be null if we failed to connect */ mutex_lock(&ep->com.mutex); + set_bit(CLOSE_CON_RPL, &ep->com.history); switch (ep->com.state) { @@ -580,13 +607,14 @@ static void process_data(struct c4iw_ep *ep) { struct sockaddr_in *local, *remote; + int disconnect = 0; CTR5(KTR_IW_CXGBE, "%s: so %p, ep %p, state %s, sbused %d", __func__, ep->com.so, ep, states[ep->com.state], sbused(&ep->com.so->so_rcv)); switch (state_read(&ep->com)) { case MPA_REQ_SENT: - process_mpa_reply(ep); + disconnect = process_mpa_reply(ep); break; case MPA_REQ_WAIT: in_getsockaddr(ep->com.so, (struct sockaddr **)&local); @@ -595,7 +623,7 @@ process_data(struct c4iw_ep *ep) ep->com.remote_addr = *remote; free(local, M_SONAME); free(remote, M_SONAME); - process_mpa_request(ep); + disconnect = process_mpa_request(ep); break; default: if (sbused(&ep->com.so->so_rcv)) @@ -605,6 +633,9 @@ process_data(struct c4iw_ep *ep) ep->com.so->so_state, sbused(&ep->com.so->so_rcv)); break; } + if (disconnect) + c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); + } static void @@ -749,9 +780,9 @@ int db_delay_usecs = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, db_delay_usecs, CTLFLAG_RWTUN, &db_delay_usecs, 0, "Usecs to delay awaiting db fifo to drain"); -static int dack_mode = 1; +static int dack_mode = 0; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, dack_mode, CTLFLAG_RWTUN, &dack_mode, 0, - "Delayed ack mode (default = 1)"); + "Delayed ack mode (default = 0)"); int c4iw_max_read_depth = 8; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_max_read_depth, CTLFLAG_RWTUN, &c4iw_max_read_depth, 0, @@ -773,9 +804,9 @@ int c4iw_debug = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, c4iw_debug, CTLFLAG_RWTUN, &c4iw_debug, 0, "Enable debug logging (default = 0)"); -static int peer2peer; +static int peer2peer = 1; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, peer2peer, CTLFLAG_RWTUN, &peer2peer, 0, - "Support peer2peer ULPs (default = 0)"); + "Support peer2peer ULPs (default = 1)"); static int p2p_type = FW_RI_INIT_P2PTYPE_READ_REQ; SYSCTL_INT(_hw_iw_cxgbe, OID_AUTO, p2p_type, CTLFLAG_RWTUN, &p2p_type, 0, @@ -827,14 +858,16 @@ start_ep_timer(struct c4iw_ep *ep) add_timer(&ep->timer); } -static void +static int stop_ep_timer(struct c4iw_ep *ep) { del_timer_sync(&ep->timer); if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { c4iw_put_ep(&ep->com); + return 0; } + return 1; } static enum @@ -900,6 +933,8 @@ void _c4iw_free_ep(struct kref *kref) epc = &ep->com; KASSERT(!epc->entry.tqe_prev, ("%s epc %p still on req list", __func__, epc)); + if (test_bit(QP_REFERENCED, &ep->com.flags)) + deref_qp(ep); kfree(ep); } @@ -1175,20 +1210,17 @@ static void close_complete_upcall(struct c4iw_ep *ep, int status) CTR2(KTR_IW_CXGBE, "%s:ccu1 %1", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); - ep->com.cm_id->rem_ref(ep->com.cm_id); - ep->com.cm_id = NULL; - ep->com.qp = NULL; + deref_cm_id(&ep->com); set_bit(CLOSE_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:ccuE %p", __func__, ep); } -static int abort_connection(struct c4iw_ep *ep) +static int send_abort(struct c4iw_ep *ep) { int err; CTR2(KTR_IW_CXGBE, "%s:abB %p", __func__, ep); - state_set(&ep->com, ABORTING); abort_socket(ep); err = close_socket(&ep->com, 0); set_bit(ABORT_CONN, &ep->com.history); @@ -1226,9 +1258,7 @@ static void peer_abort_upcall(struct c4iw_ep *ep) CTR2(KTR_IW_CXGBE, "%s:pau1 %p", __func__, ep); ep->com.cm_id->event_handler(ep->com.cm_id, &event); - ep->com.cm_id->rem_ref(ep->com.cm_id); - ep->com.cm_id = NULL; - ep->com.qp = NULL; + deref_cm_id(&ep->com); set_bit(ABORT_UPCALL, &ep->com.history); } CTR2(KTR_IW_CXGBE, "%s:pauE %p", __func__, ep); @@ -1282,9 +1312,7 @@ static void connect_reply_upcall(struct c4iw_ep *ep, int status) if (status < 0) { CTR3(KTR_IW_CXGBE, "%s:cru4 %p %d", __func__, ep, status); - ep->com.cm_id->rem_ref(ep->com.cm_id); - ep->com.cm_id = NULL; - ep->com.qp = NULL; + deref_cm_id(&ep->com); } CTR2(KTR_IW_CXGBE, "%s:cruE %p", __func__, ep); @@ -1353,8 +1381,19 @@ static void established_upcall(struct c4iw_ep *ep) } - -static void process_mpa_reply(struct c4iw_ep *ep) +/* + * process_mpa_reply - process streaming mode MPA reply + * + * Returns: + * + * 0 upon success indicating a connect request was delivered to the ULP + * or the mpa request is incomplete but valid so far. + * + * 1 if a failure requires the caller to close the connection. + * + * 2 if a failure requires the caller to abort the connection. + */ +static int process_mpa_reply(struct c4iw_ep *ep) { struct mpa_message *mpa; struct mpa_v2_conn_params *mpa_v2_params; @@ -1367,17 +1406,17 @@ static void process_mpa_reply(struct c4iw_ep *ep) struct mbuf *top, *m; int flags = MSG_DONTWAIT; struct uio uio; + int disconnect = 0; CTR2(KTR_IW_CXGBE, "%s:pmrB %p", __func__, ep); /* - * Stop mpa timer. If it expired, then the state has - * changed and we bail since ep_timeout already aborted - * the connection. + * Stop mpa timer. If it expired, then + * we ignore the MPA reply. process_timeout() + * will abort the connection. */ - STOP_EP_TIMER(ep); - if (state_read(&ep->com) != MPA_REQ_SENT) - return; + if (STOP_EP_TIMER(ep)) + return 0; uio.uio_resid = 1000000; uio.uio_td = ep->com.thread; @@ -1389,7 +1428,7 @@ static void process_mpa_reply(struct c4iw_ep *ep) CTR2(KTR_IW_CXGBE, "%s:pmr1 %p", __func__, ep); START_EP_TIMER(ep); - return; + return 0; } err = -err; CTR2(KTR_IW_CXGBE, "%s:pmr2 %p", __func__, ep); @@ -1417,7 +1456,7 @@ static void process_mpa_reply(struct c4iw_ep *ep) CTR3(KTR_IW_CXGBE, "%s:pmr5 %p %d", __func__, ep, ep->mpa_pkt_len + m->m_len); err = (-EINVAL); - goto err; + goto err_stop_timer; } /* @@ -1435,8 +1474,9 @@ static void process_mpa_reply(struct c4iw_ep *ep) /* * if we don't even have the mpa message, then bail. */ - if (ep->mpa_pkt_len < sizeof(*mpa)) - return; + if (ep->mpa_pkt_len < sizeof(*mpa)) { + return 0; + } mpa = (struct mpa_message *) ep->mpa_pkt; /* Validate MPA header. */ @@ -1447,14 +1487,14 @@ static void process_mpa_reply(struct c4iw_ep *ep) printk(KERN_ERR MOD "%s MPA version mismatch. Local = %d, " " Received = %d\n", __func__, mpa_rev, mpa->revision); err = -EPROTO; - goto err; + goto err_stop_timer; } if (memcmp(mpa->key, MPA_KEY_REP, sizeof(mpa->key))) { CTR2(KTR_IW_CXGBE, "%s:pmr7 %p", __func__, ep); err = -EPROTO; - goto err; + goto err_stop_timer; } plen = ntohs(mpa->private_data_size); @@ -1466,7 +1506,7 @@ static void process_mpa_reply(struct c4iw_ep *ep) CTR2(KTR_IW_CXGBE, "%s:pmr8 %p", __func__, ep); err = -EPROTO; - goto err; + goto err_stop_timer; } /* @@ -1475,8 +1515,9 @@ static void process_mpa_reply(struct c4iw_ep *ep) if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) { CTR2(KTR_IW_CXGBE, "%s:pmr9 %p", __func__, ep); + STOP_EP_TIMER(ep); err = -EPROTO; - goto err; + goto err_stop_timer; } ep->plen = (u8) plen; @@ -1488,14 +1529,14 @@ static void process_mpa_reply(struct c4iw_ep *ep) if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) { CTR2(KTR_IW_CXGBE, "%s:pmra %p", __func__, ep); - return; + return 0; } if (mpa->flags & MPA_REJECT) { CTR2(KTR_IW_CXGBE, "%s:pmrb %p", __func__, ep); err = -ECONNREFUSED; - goto err; + goto err_stop_timer; } /* @@ -1638,6 +1679,7 @@ static void process_mpa_reply(struct c4iw_ep *ep) err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); err = -ENOMEM; + disconnect = 1; goto out; } @@ -1658,19 +1700,33 @@ static void process_mpa_reply(struct c4iw_ep *ep) err = c4iw_modify_qp(ep->com.qp->rhp, ep->com.qp, C4IW_QP_ATTR_NEXT_STATE, &attrs, 0); err = -ENOMEM; + disconnect = 1; goto out; } goto out; +err_stop_timer: + STOP_EP_TIMER(ep); err: - state_set(&ep->com, ABORTING); - abort_connection(ep); + disconnect = 2; out: connect_reply_upcall(ep, err); CTR2(KTR_IW_CXGBE, "%s:pmrE %p", __func__, ep); - return; + return disconnect; } -static void +/* + * process_mpa_request - process streaming mode MPA request + * + * Returns: + * + * 0 upon success indicating a connect request was delivered to the ULP + * or the mpa request is incomplete but valid so far. + * + * 1 if a failure requires the caller to close the connection. + * + * 2 if a failure requires the caller to abort the connection. + */ +static int process_mpa_request(struct c4iw_ep *ep) { struct mpa_message *mpa; @@ -1684,7 +1740,7 @@ process_mpa_request(struct c4iw_ep *ep) CTR3(KTR_IW_CXGBE, "%s: ep %p, state %s", __func__, ep, states[state]); if (state != MPA_REQ_WAIT) - return; + return 0; iov.iov_base = &ep->mpa_pkt[ep->mpa_pkt_len]; iov.iov_len = sizeof(ep->mpa_pkt) - ep->mpa_pkt_len; @@ -1698,13 +1754,10 @@ process_mpa_request(struct c4iw_ep *ep) rc = soreceive(ep->com.so, NULL, &uio, NULL, NULL, &flags); if (rc == EAGAIN) - return; - else if (rc) { -abort: - STOP_EP_TIMER(ep); - abort_connection(ep); - return; - } + return 0; + else if (rc) + goto err_stop_timer; + KASSERT(uio.uio_offset > 0, ("%s: sorecieve on so %p read no data", __func__, ep->com.so)); ep->mpa_pkt_len += uio.uio_offset; @@ -1718,7 +1771,7 @@ abort: /* Don't even have the MPA message. Wait for more data to arrive. */ if (ep->mpa_pkt_len < sizeof(*mpa)) - return; + return 0; mpa = (struct mpa_message *) ep->mpa_pkt; /* @@ -1727,24 +1780,24 @@ abort: if (mpa->revision > mpa_rev) { log(LOG_ERR, "%s: MPA version mismatch. Local = %d," " Received = %d\n", __func__, mpa_rev, mpa->revision); - goto abort; + goto err_stop_timer; } if (memcmp(mpa->key, MPA_KEY_REQ, sizeof(mpa->key))) - goto abort; + goto err_stop_timer; /* * Fail if there's too much private data. */ plen = ntohs(mpa->private_data_size); if (plen > MPA_MAX_PRIVATE_DATA) - goto abort; + goto err_stop_timer; /* * If plen does not account for pkt size */ if (ep->mpa_pkt_len > (sizeof(*mpa) + plen)) - goto abort; + goto err_stop_timer; ep->plen = (u8) plen; @@ -1752,7 +1805,7 @@ abort: * If we don't have all the pdata yet, then bail. */ if (ep->mpa_pkt_len < (sizeof(*mpa) + plen)) - return; + return 0; /* * If we get here we have accumulated the entire mpa @@ -1794,7 +1847,7 @@ abort: ep->mpa_attr.p2p_type = p2p_type; if (set_tcpinfo(ep)) - goto abort; + goto err_stop_timer; CTR5(KTR_IW_CXGBE, "%s: crc_enabled = %d, recv_marker_enabled = %d, " "xmit_marker_enabled = %d, version = %d", __func__, @@ -1802,17 +1855,22 @@ abort: ep->mpa_attr.xmit_marker_enabled, ep->mpa_attr.version); state_set(&ep->com, MPA_REQ_RCVD); - STOP_EP_TIMER(ep); /* drive upcall */ mutex_lock(&ep->parent_ep->com.mutex); if (ep->parent_ep->com.state != DEAD) { - if(connect_request_upcall(ep)) { - abort_connection(ep); - } - }else - abort_connection(ep); + if(connect_request_upcall(ep)) + goto err_out; + }else { + goto err_out; + } mutex_unlock(&ep->parent_ep->com.mutex); + return 0; + +err_stop_timer: + STOP_EP_TIMER(ep); +err_out: + return 2; } /* @@ -1825,6 +1883,7 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) int err; struct c4iw_ep *ep = to_ep(cm_id); CTR2(KTR_IW_CXGBE, "%s:crcB %p", __func__, ep); + int disconnect = 0; if (state_read(&ep->com) == DEAD) { @@ -1838,7 +1897,7 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) if (mpa_rev == 0) { CTR2(KTR_IW_CXGBE, "%s:crc2 %p", __func__, ep); - abort_connection(ep); + disconnect = 2; } else { @@ -1847,6 +1906,8 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) err = soshutdown(ep->com.so, 3); } c4iw_put_ep(&ep->com); + if (disconnect) + err = c4iw_ep_disconnect(ep, disconnect == 2, GFP_KERNEL); CTR2(KTR_IW_CXGBE, "%s:crc4 %p", __func__, ep); return 0; } @@ -1859,6 +1920,7 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) struct c4iw_ep *ep = to_ep(cm_id); struct c4iw_dev *h = to_c4iw_dev(cm_id->device); struct c4iw_qp *qp = get_qhp(h, conn_param->qpn); + int abort = 0; CTR2(KTR_IW_CXGBE, "%s:cacB %p", __func__, ep); @@ -1866,7 +1928,7 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) CTR2(KTR_IW_CXGBE, "%s:cac1 %p", __func__, ep); err = -ECONNRESET; - goto err; + goto err_out; } BUG_ON(state_read(&ep->com) != MPA_REQ_RCVD); @@ -1878,9 +1940,8 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) (conn_param->ird > c4iw_max_read_depth)) { CTR2(KTR_IW_CXGBE, "%s:cac2 %p", __func__, ep); - abort_connection(ep); err = -EINVAL; - goto err; + goto err_abort; } if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { @@ -1894,9 +1955,8 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ep->ord = conn_param->ord; send_mpa_reject(ep, conn_param->private_data, conn_param->private_data_len); - abort_connection(ep); err = -ENOMEM; - goto err; + goto err_abort; } if (conn_param->ird > ep->ord) { @@ -1910,9 +1970,8 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) } else { CTR2(KTR_IW_CXGBE, "%s:cac7 %p", __func__, ep); - abort_connection(ep); err = -ENOMEM; - goto err; + goto err_abort; } } @@ -1932,9 +1991,10 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) } - cm_id->add_ref(cm_id); ep->com.cm_id = cm_id; + ref_cm_id(&ep->com); ep->com.qp = qp; + ref_qp(ep); //ep->ofld_txq = TOEPCB(ep->com.so)->ofld_txq; /* bind QP to EP and move to RTS */ @@ -1956,7 +2016,7 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) if (err) { CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); - goto err1; + goto err_defef_cm_id; } err = send_mpa_reply(ep, conn_param->private_data, conn_param->private_data_len); @@ -1964,7 +2024,7 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) if (err) { CTR2(KTR_IW_CXGBE, "%s:caca %p", __func__, ep); - goto err1; + goto err_defef_cm_id; } state_set(&ep->com, FPDU_MODE); @@ -1972,11 +2032,13 @@ int c4iw_accept_cr(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:cacE %p", __func__, ep); return 0; -err1: - ep->com.cm_id = NULL; - ep->com.qp = NULL; - cm_id->rem_ref(cm_id); -err: +err_defef_cm_id: + deref_cm_id(&ep->com); +err_abort: + abort = 1; +err_out: + if (abort) + c4iw_ep_disconnect(ep, 1, GFP_KERNEL); c4iw_put_ep(&ep->com); CTR2(KTR_IW_CXGBE, "%s:cacE err %p", __func__, ep); return err; @@ -2028,9 +2090,9 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) ep->ord = 1; } - cm_id->add_ref(cm_id); ep->com.dev = dev; ep->com.cm_id = cm_id; + ref_cm_id(&ep->com); ep->com.qp = get_qhp(dev, conn_param->qpn); if (!ep->com.qp) { @@ -2039,6 +2101,7 @@ int c4iw_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) err = -EINVAL; goto fail2; } + ref_qp(ep); ep->com.thread = curthread; ep->com.so = cm_id->so; @@ -2096,7 +2159,7 @@ fail3: CTR2(KTR_IW_CXGBE, "%s:ccb %p", __func__, ep); fib4_free_nh_ext(RT_DEFAULT_FIB, &nh4); fail2: - cm_id->rem_ref(cm_id); + deref_cm_id(&ep->com); c4iw_put_ep(&ep->com); out: CTR2(KTR_IW_CXGBE, "%s:ccE %p", __func__, ep); @@ -2124,8 +2187,8 @@ c4iw_create_listen_ep(struct iw_cm_id *cm_id, int backlog) goto failed; } - cm_id->add_ref(cm_id); ep->com.cm_id = cm_id; + ref_cm_id(&ep->com); ep->com.dev = dev; ep->backlog = backlog; ep->com.local_addr = cm_id->local_addr; @@ -2150,7 +2213,7 @@ c4iw_destroy_listen_ep(struct iw_cm_id *cm_id) cm_id->so, states[ep->com.state]); state_set(&ep->com, DEAD); - cm_id->rem_ref(cm_id); + deref_cm_id(&ep->com); c4iw_put_ep(&ep->com); return; @@ -2232,7 +2295,8 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) CTR2(KTR_IW_CXGBE, "%s:ced4 %p", __func__, ep); set_bit(EP_DISC_ABORT, &ep->com.history); - ret = abort_connection(ep); + close_complete_upcall(ep, -ECONNRESET); + ret = send_abort(ep); } else { CTR2(KTR_IW_CXGBE, "%s:ced5 %p", __func__, ep); @@ -2250,8 +2314,25 @@ int c4iw_ep_disconnect(struct c4iw_ep *ep, int abrupt, gfp_t gfp) } if (fatal) { + set_bit(EP_DISC_FAIL, &ep->com.history); + if (!abrupt) { + STOP_EP_TIMER(ep); + close_complete_upcall(ep, -EIO); + } + if (ep->com.qp) { + struct c4iw_qp_attributes attrs; + attrs.next_state = C4IW_QP_STATE_ERROR; + ret = c4iw_modify_qp(ep->com.dev, ep->com.qp, + C4IW_QP_ATTR_NEXT_STATE, + &attrs, 1); + if (ret) { + CTR2(KTR_IW_CXGBE, "%s:ced7 %p", __func__, ep); + printf("%s - qp <- error failed!\n", __func__); + } + } release_ep_resources(ep); + ep->com.state = DEAD; CTR2(KTR_IW_CXGBE, "%s:ced6 %p", __func__, ep); } CTR2(KTR_IW_CXGBE, "%s:cedE %p", __func__, ep); @@ -2290,8 +2371,13 @@ static void ep_timeout(unsigned long arg) if (!test_and_set_bit(TIMEOUT, &ep->com.flags)) { - list_add_tail(&ep->entry, &timeout_list); - kickit = 1; + /* + * Only insert if it is not already on the list. + */ + if (!ep->entry.next) { + list_add_tail(&ep->entry, &timeout_list); + kickit = 1; + } } spin_unlock(&timeout_lock); diff --git a/sys/dev/cxgbe/iw_cxgbe/cq.c b/sys/dev/cxgbe/iw_cxgbe/cq.c index 8710e03a829c..b40ffc7fb6c1 100644 --- a/sys/dev/cxgbe/iw_cxgbe/cq.c +++ b/sys/dev/cxgbe/iw_cxgbe/cq.c @@ -724,7 +724,7 @@ static int c4iw_poll_cq_one(struct c4iw_cq *chp, struct ib_wc *wc) default: printf("Unexpected cqe_status 0x%x for QPID = 0x%0x\n", CQE_STATUS(&cqe), CQE_QPID(&cqe)); - ret = -EINVAL; + wc->status = IB_WC_FATAL_ERR; } } out: @@ -861,6 +861,7 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr, if (!mm2) goto err4; + memset(&uresp, 0, sizeof(uresp)); uresp.qid_mask = rhp->rdev.cqmask; uresp.cqid = chp->cq.cqid; uresp.size = chp->cq.size; @@ -871,7 +872,8 @@ c4iw_create_cq(struct ib_device *ibdev, struct ib_cq_init_attr *attr, uresp.gts_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); + ret = ib_copy_to_udata(udata, &uresp, + sizeof(uresp) - sizeof(uresp.reserved)); if (ret) goto err5; diff --git a/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h b/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h index c232f70bd254..38a8af6f0819 100644 --- a/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h +++ b/sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h @@ -157,7 +157,7 @@ static inline int c4iw_fatal_error(struct c4iw_rdev *rdev) static inline int c4iw_num_stags(struct c4iw_rdev *rdev) { - return min((int)T4_MAX_NUM_STAG, (int)(rdev->adap->vres.stag.size >> 5)); + return (int)(rdev->adap->vres.stag.size >> 5); } #define C4IW_WR_TO (10*HZ) @@ -435,6 +435,7 @@ struct c4iw_qp { atomic_t refcnt; wait_queue_head_t wait; struct timer_list timer; + int sq_sig_all; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) @@ -712,7 +713,8 @@ enum c4iw_ep_flags { ABORT_REQ_IN_PROGRESS = 1, RELEASE_RESOURCES = 2, CLOSE_SENT = 3, - TIMEOUT = 4 + TIMEOUT = 4, + QP_REFERENCED = 5 }; enum c4iw_ep_history { @@ -737,7 +739,13 @@ enum c4iw_ep_history { EP_DISC_ABORT = 18, CONN_RPL_UPCALL = 19, ACT_RETRY_NOMEM = 20, - ACT_RETRY_INUSE = 21 + ACT_RETRY_INUSE = 21, + CLOSE_CON_RPL = 22, + EP_DISC_FAIL = 24, + QP_REFED = 25, + QP_DEREFED = 26, + CM_ID_REFED = 27, + CM_ID_DEREFED = 28 }; struct c4iw_ep_common { diff --git a/sys/dev/cxgbe/iw_cxgbe/mem.c b/sys/dev/cxgbe/iw_cxgbe/mem.c index f7c460ac5cac..e42aa1a4b58a 100644 --- a/sys/dev/cxgbe/iw_cxgbe/mem.c +++ b/sys/dev/cxgbe/iw_cxgbe/mem.c @@ -46,6 +46,12 @@ __FBSDID("$FreeBSD$"); #define T4_ULPTX_MIN_IO 32 #define C4IW_MAX_INLINE_SIZE 96 +static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length) +{ + return (is_t4(dev->rdev.adap) || + is_t5(dev->rdev.adap)) && + length >= 8*1024*1024*1024ULL; +} static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data) { @@ -144,8 +150,12 @@ static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry, if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) { stag_idx = c4iw_get_resource(&rdev->resource.tpt_table); - if (!stag_idx) + if (!stag_idx) { + mutex_lock(&rdev->stats.lock); + rdev->stats.stag.fail++; + mutex_unlock(&rdev->stats.lock); return -ENOMEM; + } mutex_lock(&rdev->stats.lock); rdev->stats.stag.cur += 32; if (rdev->stats.stag.cur > rdev->stats.stag.max) @@ -250,9 +260,9 @@ static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php, int ret; ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid, - FW_RI_STAG_NSMR, mhp->attr.perms, + FW_RI_STAG_NSMR, mhp->attr.len ? mhp->attr.perms : 0, mhp->attr.mw_bind_enable, mhp->attr.zbva, - mhp->attr.va_fbo, mhp->attr.len, shift - 12, + mhp->attr.va_fbo, mhp->attr.len ? mhp->attr.len : -1, shift - 12, mhp->attr.pbl_size, mhp->attr.pbl_addr); if (ret) return ret; @@ -381,7 +391,7 @@ int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, struct c4iw_dev *rhp; __be64 *page_list = NULL; int shift = 0; - u64 total_size; + u64 total_size = 0; int npages = 0; int ret; @@ -416,7 +426,10 @@ int c4iw_reregister_phys_mem(struct ib_mr *mr, int mr_rereg_mask, if (ret) return ret; } - + if (mr_exceeds_hw_limits(rhp, total_size)) { + kfree(page_list); + return -EINVAL; + } ret = reregister_mem(rhp, php, &mh, shift, npages); kfree(page_list); if (ret) @@ -477,10 +490,15 @@ struct ib_mr *c4iw_register_phys_mem(struct ib_pd *pd, if (ret) goto err; + if (mr_exceeds_hw_limits(rhp, total_size)) { + kfree(page_list); + ret = -EINVAL; + goto err; + } ret = alloc_pbl(mhp, npages); if (ret) { kfree(page_list); - goto err_pbl; + goto err; } ret = write_pbl(&mhp->rhp->rdev, page_list, mhp->attr.pbl_addr, @@ -580,6 +598,10 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, php = to_c4iw_pd(pd); rhp = php->rhp; + + if (mr_exceeds_hw_limits(rhp, length)) + return ERR_PTR(-EINVAL); + mhp = kzalloc(sizeof(*mhp), GFP_KERNEL); if (!mhp) return ERR_PTR(-ENOMEM); diff --git a/sys/dev/cxgbe/iw_cxgbe/qp.c b/sys/dev/cxgbe/iw_cxgbe/qp.c index 1c0381c6f59e..3ee16b1f8022 100644 --- a/sys/dev/cxgbe/iw_cxgbe/qp.c +++ b/sys/dev/cxgbe/iw_cxgbe/qp.c @@ -615,7 +615,7 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, fw_flags = 0; if (wr->send_flags & IB_SEND_SOLICITED) fw_flags |= FW_RI_SOLICITED_EVENT_FLAG; - if (wr->send_flags & IB_SEND_SIGNALED) + if (wr->send_flags & IB_SEND_SIGNALED || qhp->sq_sig_all) fw_flags |= FW_RI_COMPLETION_FLAG; swsqe = &qhp->wq.sq.sw_sq[qhp->wq.sq.pidx]; switch (wr->opcode) { @@ -673,7 +673,8 @@ int c4iw_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, } swsqe->idx = qhp->wq.sq.pidx; swsqe->complete = 0; - swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED); + swsqe->signaled = (wr->send_flags & IB_SEND_SIGNALED) || + qhp->sq_sig_all; swsqe->wr_id = wr->wr_id; init_wr_hdr(wqe, qhp->wq.sq.pidx, fw_opcode, fw_flags, len16); @@ -952,7 +953,7 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, flushed = c4iw_flush_rq(&qhp->wq, &rchp->cq, count); spin_unlock(&qhp->lock); spin_unlock_irqrestore(&rchp->lock, flag); - if (flushed) { + if (flushed && rchp->ibcq.comp_handler) { spin_lock_irqsave(&rchp->comp_handler_lock, flag); (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); spin_unlock_irqrestore(&rchp->comp_handler_lock, flag); @@ -966,7 +967,7 @@ static void __flush_qp(struct c4iw_qp *qhp, struct c4iw_cq *rchp, flushed = c4iw_flush_sq(&qhp->wq, &schp->cq, count); spin_unlock(&qhp->lock); spin_unlock_irqrestore(&schp->lock, flag); - if (flushed) { + if (flushed && schp->ibcq.comp_handler) { spin_lock_irqsave(&schp->comp_handler_lock, flag); (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); spin_unlock_irqrestore(&schp->comp_handler_lock, flag); @@ -1530,6 +1531,7 @@ c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, qhp->attr.enable_bind = 1; qhp->attr.max_ord = 1; qhp->attr.max_ird = 1; + qhp->sq_sig_all = attrs->sq_sig_type == IB_SIGNAL_ALL_WR; spin_lock_init(&qhp->lock); mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); @@ -1702,6 +1704,12 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, memset(attr, 0, sizeof *attr); memset(init_attr, 0, sizeof *init_attr); attr->qp_state = to_ib_qp_state(qhp->attr.state); + init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; + init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; + init_attr->cap.max_send_sge = qhp->attr.sq_max_sges; + init_attr->cap.max_recv_sge = qhp->attr.sq_max_sges; + init_attr->cap.max_inline_data = T4_MAX_SEND_INLINE; + init_attr->sq_sig_type = qhp->sq_sig_all ? IB_SIGNAL_ALL_WR : 0; return 0; } #endif diff --git a/sys/dev/cxgbe/iw_cxgbe/t4.h b/sys/dev/cxgbe/iw_cxgbe/t4.h index 023c607f8bfb..b0118f9906ba 100644 --- a/sys/dev/cxgbe/iw_cxgbe/t4.h +++ b/sys/dev/cxgbe/iw_cxgbe/t4.h @@ -69,7 +69,6 @@ #define T4_MAX_SQ_SIZE (T4_MAX_EQ_SIZE - 1) #define T4_MAX_QP_DEPTH (T4_MAX_RQ_SIZE - 1) #define T4_MAX_CQ_DEPTH (T4_MAX_IQ_SIZE - 1) -#define T4_MAX_NUM_STAG (1<<15) #define T4_MAX_MR_SIZE (~0ULL - 1) #define T4_PAGESIZE_MASK 0xffff000 /* 4KB-128MB */ #define T4_STAG_UNSET 0xffffffff @@ -524,7 +523,7 @@ static inline void t4_swcq_consume(struct t4_cq *cq) static inline void t4_hwcq_consume(struct t4_cq *cq) { cq->bits_type_ts = cq->queue[cq->cidx].bits_type_ts; - if (++cq->cidx_inc == (cq->size >> 4)) { + if (++cq->cidx_inc == (cq->size >> 4) || cq->cidx_inc == M_CIDXINC) { u32 val; val = SEINTARM(0) | CIDXINC(cq->cidx_inc) | TIMERREG(7) | diff --git a/sys/dev/cxgbe/iw_cxgbe/user.h b/sys/dev/cxgbe/iw_cxgbe/user.h index 59a1f43642f7..d42f659771ae 100644 --- a/sys/dev/cxgbe/iw_cxgbe/user.h +++ b/sys/dev/cxgbe/iw_cxgbe/user.h @@ -50,6 +50,7 @@ struct c4iw_create_cq_resp { __u32 cqid; __u32 size; __u32 qid_mask; + __u32 reserved; /* explicit padding (optional for i386) */ }; struct c4iw_create_qp_resp {