cxgbe/cxgbei: Count various events related to iSCSI operation and make

these counters available in the sysctl MIB.

Sponsored by:	Chelsio Communications

sys/dev/cxgbe/cxgbei/cxgbei.c

@@ -100,6 +100,50 @@ struct icl_pdu *icl_cxgbei_new_pdu(int);
 void icl_cxgbei_new_pdu_set_conn(struct icl_pdu *, struct icl_conn *);
 void icl_cxgbei_conn_pdu_free(struct icl_conn *, struct icl_pdu *);
 
+static void
+free_ci_counters(struct cxgbei_data *ci)
+{
+
+#define FREE_CI_COUNTER(x) do { \
+	if (ci->x != NULL) { \
+		counter_u64_free(ci->x); \
+		ci->x = NULL; \
+	} \
+} while (0)
+
+	FREE_CI_COUNTER(ddp_setup_ok);
+	FREE_CI_COUNTER(ddp_setup_error);
+	FREE_CI_COUNTER(ddp_bytes);
+	FREE_CI_COUNTER(ddp_pdus);
+	FREE_CI_COUNTER(fl_bytes);
+	FREE_CI_COUNTER(fl_pdus);
+#undef FREE_CI_COUNTER
+}
+
+static int
+alloc_ci_counters(struct cxgbei_data *ci)
+{
+
+#define ALLOC_CI_COUNTER(x) do { \
+	ci->x = counter_u64_alloc(M_WAITOK); \
+	if (ci->x == NULL) \
+		goto fail; \
+} while (0)
+
+	ALLOC_CI_COUNTER(ddp_setup_ok);
+	ALLOC_CI_COUNTER(ddp_setup_error);
+	ALLOC_CI_COUNTER(ddp_bytes);
+	ALLOC_CI_COUNTER(ddp_pdus);
+	ALLOC_CI_COUNTER(fl_bytes);
+	ALLOC_CI_COUNTER(fl_pdus);
+#undef ALLOC_CI_COUNTER
+
+	return (0);
+fail:
+	free_ci_counters(ci);
+	return (ENOMEM);
+}
+
 static void
 read_pdu_limits(struct adapter *sc, uint32_t *max_tx_pdu_len,
     uint32_t *max_rx_pdu_len)
@@ -133,6 +177,8 @@ read_pdu_limits(struct adapter *sc, uint32_t *max_tx_pdu_len,
 static int
 cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
 {
+	struct sysctl_oid *oid;
+	struct sysctl_oid_list *children;
 	struct ppod_region *pr;
 	uint32_t r;
 	int rc;
@@ -140,6 +186,10 @@ cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
 	MPASS(sc->vres.iscsi.size > 0);
 	MPASS(ci != NULL);
 
+	rc = alloc_ci_counters(ci);
+	if (rc != 0)
+		return (rc);
+
 	read_pdu_limits(sc, &ci->max_tx_pdu_len, &ci->max_rx_pdu_len);
 
 	ci->ddp_threshold = 2048;
@@ -151,6 +201,7 @@ cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
 		device_printf(sc->dev,
 		    "%s: failed to initialize the iSCSI page pod region: %u.\n",
 		    __func__, rc);
+		free_ci_counters(ci);
 		return (rc);
 	}
 
@@ -169,6 +220,35 @@ cxgbei_init(struct adapter *sc, struct cxgbei_data *ci)
 		    V_ISCSITAGMASK(M_ISCSITAGMASK), pr->pr_tag_mask);
 	}
 
+	sysctl_ctx_init(&ci->ctx);
+	oid = device_get_sysctl_tree(sc->dev);	/* dev.t5nex.X */
+	children = SYSCTL_CHILDREN(oid);
+
+	oid = SYSCTL_ADD_NODE(&ci->ctx, children, OID_AUTO, "iscsi", CTLFLAG_RD,
+	    NULL, "iSCSI ULP statistics");
+	children = SYSCTL_CHILDREN(oid);
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_setup_ok",
+	    CTLFLAG_RD, &ci->ddp_setup_ok,
+	    "# of times DDP buffer was setup successfully.");
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_setup_error",
+	    CTLFLAG_RD, &ci->ddp_setup_error,
+	    "# of times DDP buffer setup failed.");
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_bytes",
+	    CTLFLAG_RD, &ci->ddp_bytes, "# of bytes placed directly");
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "ddp_pdus",
+	    CTLFLAG_RD, &ci->ddp_pdus, "# of PDUs with data placed directly.");
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "fl_bytes",
+	    CTLFLAG_RD, &ci->fl_bytes, "# of data bytes delivered in freelist");
+
+	SYSCTL_ADD_COUNTER_U64(&ci->ctx, children, OID_AUTO, "fl_pdus",
+	    CTLFLAG_RD, &ci->fl_pdus,
+	    "# of PDUs with data delivered in freelist");
+
 	return (0);
 }
 
@@ -213,6 +293,7 @@ static int
 do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
+	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
 	struct cpl_iscsi_data *cpl =  mtod(m, struct cpl_iscsi_data *);
 	u_int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
@@ -232,6 +313,8 @@ do_rx_iscsi_data(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m
 
 	icp->icp_flags |= ICPF_RX_FLBUF;
 	icp->ip.ip_data_mbuf = m;
+	counter_u64_add(ci->fl_pdus, 1);
+	counter_u64_add(ci->fl_bytes, m->m_pkthdr.len);
 
 #if 0
 	CTR3(KTR_CXGBE, "%s: tid %u, cpl->len %u", __func__, tid,
@@ -245,6 +328,7 @@ static int
 do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 {
 	struct adapter *sc = iq->adapter;
+	struct cxgbei_data *ci = sc->iscsi_ulp_softc;
 	const struct cpl_rx_data_ddp *cpl = (const void *)(rss + 1);
 	u_int tid = GET_TID(cpl);
 	struct toepcb *toep = lookup_tid(sc, tid);
@@ -286,6 +370,8 @@ do_rx_iscsi_ddp(struct sge_iq *iq, const struct rss_header *rss, struct mbuf *m)
 		MPASS((icp->icp_flags & ICPF_RX_FLBUF) == 0);
 		MPASS(ip->ip_data_len > 0);
 		icp->icp_flags |= ICPF_RX_DDP;
+		counter_u64_add(ci->ddp_pdus, 1);
+		counter_u64_add(ci->ddp_bytes, ip->ip_data_len);
 	}
 
 	INP_WLOCK(inp);
@@ -439,7 +525,9 @@ cxgbei_deactivate(struct adapter *sc)
 	ASSERT_SYNCHRONIZED_OP(sc);
 
 	if (ci != NULL) {
+		sysctl_ctx_free(&ci->ctx);
 		t4_free_ppod_region(&ci->pr);
+		free_ci_counters(ci);
 		free(ci, M_CXGBE);
 		sc->iscsi_ulp_softc = NULL;
 	}

sys/dev/cxgbe/cxgbei/cxgbei.h

@@ -112,6 +112,13 @@ struct cxgbei_data {
 	u_int ddp_threshold;
 	struct ppod_region pr;
 
+	struct sysctl_ctx_list ctx;	/* from uld_activate to deactivate */
+	counter_u64_t ddp_setup_ok;
+	counter_u64_t ddp_setup_error;
+	counter_u64_t ddp_bytes;
+	counter_u64_t ddp_pdus;
+	counter_u64_t fl_bytes;
+	counter_u64_t fl_pdus;
 };
 
 /* cxgbei.c */

sys/dev/cxgbe/cxgbei/icl_cxgbei.c

@@ -820,6 +820,8 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
 		itt = V_PPOD_TAG(itt) | pr->pr_invalid_bit;
 		*ittp = htobe32(itt);
 		MPASS(*arg == NULL);	/* State is maintained for DDP only. */
+		if (rc != 0)
+			counter_u64_add(ci->ddp_setup_error, 1);
 		return (0);
 	}
 
@@ -853,6 +855,7 @@ icl_cxgbei_conn_task_setup(struct icl_conn *ic, struct icl_pdu *ip,
 
 	*ittp = htobe32(prsv->prsv_tag);
 	*arg = prsv;
+	counter_u64_add(ci->ddp_setup_ok, 1);
 	return (0);
 }
 
@@ -920,6 +923,8 @@ icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
 			ttt = V_PPOD_TAG(ttt) | pr->pr_invalid_bit;
 			*tttp = htobe32(ttt);
 			MPASS(io_to_ppod_reservation(io) == NULL);
+			if (rc != 0)
+				counter_u64_add(ci->ddp_setup_error, 1);
 			return (0);
 		}
 
@@ -965,6 +970,7 @@ icl_cxgbei_conn_transfer_setup(struct icl_conn *ic, union ctl_io *io,
 		*tttp = htobe32(prsv->prsv_tag);
 		io_to_ppod_reservation(io) = prsv;
 		*arg = ctsio;
+		counter_u64_add(ci->ddp_setup_ok, 1);
 		return (0);
 	}