127 lines
4.3 KiB
Diff
127 lines
4.3 KiB
Diff
From 6f9c7b5c83a832884b3e47a6ce52145991073586 Mon Sep 17 00:00:00 2001
|
|
From: Eli Cohen <eli@mellanox.co.il>
|
|
Date: Sun, 19 Sep 2010 11:28:40 +0200
|
|
Subject: [PATCH] libmlx4: fix possible inline size
|
|
|
|
The current driver checks required inline size by making sure it does not
|
|
exceed 1024. This is wrong since the whole WQE is limited to 1008 bytes.
|
|
Moreover, a more careful claculation is required to avoid cases where the
|
|
application requests inline support in a certain size that when used later
|
|
could cause connections to stall due to bad WQEs. This patch takes into account
|
|
the size of the WQE, the segements used to create a WQE and the overhead
|
|
incured by the inline segments themselves.
|
|
|
|
Signed-off-by: Eli Cohen <eli@mellanox.co.il>
|
|
---
|
|
src/verbs.c | 45 ++++++++++++++++++++++++++++++++++++++++-----
|
|
1 files changed, 40 insertions(+), 5 deletions(-)
|
|
|
|
Index: libmlx4/src/verbs.c
|
|
===================================================================
|
|
--- libmlx4.orig/src/verbs.c 2010-09-29 11:10:17.691587848 +0200
|
|
+++ libmlx4/src/verbs.c 2010-09-29 11:16:11.031586721 +0200
|
|
@@ -402,6 +402,44 @@ int mlx4_destroy_srq(struct ibv_srq *ibs
|
|
return 0;
|
|
}
|
|
|
|
+static int verify_sizes(struct ibv_qp_init_attr *attr, struct mlx4_context *context)
|
|
+{
|
|
+ int size;
|
|
+ int nsegs;
|
|
+
|
|
+ if (attr->cap.max_send_wr > context->max_qp_wr ||
|
|
+ attr->cap.max_recv_wr > context->max_qp_wr ||
|
|
+ attr->cap.max_send_sge > context->max_sge ||
|
|
+ attr->cap.max_recv_sge > context->max_sge)
|
|
+ return -1;
|
|
+
|
|
+ if (attr->cap.max_inline_data) {
|
|
+ nsegs = num_inline_segs(attr->cap.max_inline_data, attr->qp_type);
|
|
+ size = MLX4_MAX_WQE_SIZE - nsegs * sizeof (struct mlx4_wqe_inline_seg);
|
|
+ switch (attr->qp_type) {
|
|
+ case IBV_QPT_UD:
|
|
+ size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
|
|
+ sizeof (struct mlx4_wqe_datagram_seg));
|
|
+ break;
|
|
+
|
|
+ case IBV_QPT_RC:
|
|
+ case IBV_QPT_UC:
|
|
+ case IBV_QPT_XRC:
|
|
+ size -= (sizeof (struct mlx4_wqe_ctrl_seg) +
|
|
+ sizeof (struct mlx4_wqe_raddr_seg));
|
|
+ break;
|
|
+
|
|
+ default:
|
|
+ return 0;
|
|
+ }
|
|
+
|
|
+ if (attr->cap.max_inline_data > size)
|
|
+ return -1;
|
|
+ }
|
|
+
|
|
+ return 0;
|
|
+}
|
|
+
|
|
struct ibv_qp *mlx4_create_qp(struct ibv_pd *pd, struct ibv_qp_init_attr *attr)
|
|
{
|
|
struct mlx4_create_qp cmd;
|
|
@@ -412,11 +450,7 @@ struct ibv_qp *mlx4_create_qp(struct ibv
|
|
|
|
|
|
/* Sanity check QP size before proceeding */
|
|
- if (attr->cap.max_send_wr > context->max_qp_wr ||
|
|
- attr->cap.max_recv_wr > context->max_qp_wr ||
|
|
- attr->cap.max_send_sge > context->max_sge ||
|
|
- attr->cap.max_recv_sge > context->max_sge ||
|
|
- attr->cap.max_inline_data > 1024)
|
|
+ if (verify_sizes(attr, context))
|
|
return NULL;
|
|
|
|
qp = malloc(sizeof *qp);
|
|
Index: libmlx4/src/mlx4.h
|
|
===================================================================
|
|
--- libmlx4.orig/src/mlx4.h 2010-09-29 11:10:17.691587848 +0200
|
|
+++ libmlx4/src/mlx4.h 2010-09-29 11:11:35.559586971 +0200
|
|
@@ -159,6 +159,10 @@ enum {
|
|
MLX4_CQE_OPCODE_RESIZE = 0x16,
|
|
};
|
|
|
|
+enum {
|
|
+ MLX4_MAX_WQE_SIZE = 1008
|
|
+};
|
|
+
|
|
struct mlx4_device {
|
|
struct ibv_device ibv_dev;
|
|
int page_size;
|
|
@@ -410,6 +414,7 @@ int mlx4_post_recv(struct ibv_qp *ibqp,
|
|
struct ibv_recv_wr **bad_wr);
|
|
void mlx4_calc_sq_wqe_size(struct ibv_qp_cap *cap, enum ibv_qp_type type,
|
|
struct mlx4_qp *qp);
|
|
+int num_inline_segs(int data, enum ibv_qp_type type);
|
|
int mlx4_alloc_qp_buf(struct ibv_pd *pd, struct ibv_qp_cap *cap,
|
|
enum ibv_qp_type type, struct mlx4_qp *qp);
|
|
void mlx4_set_sq_sizes(struct mlx4_qp *qp, struct ibv_qp_cap *cap,
|
|
Index: libmlx4/src/qp.c
|
|
===================================================================
|
|
--- libmlx4.orig/src/qp.c 2010-09-29 11:10:17.691587848 +0200
|
|
+++ libmlx4/src/qp.c 2010-09-29 11:12:14.931587492 +0200
|
|
@@ -505,7 +505,7 @@ out:
|
|
return ret;
|
|
}
|
|
|
|
-static int num_inline_segs(int data, enum ibv_qp_type type)
|
|
+int num_inline_segs(int data, enum ibv_qp_type type)
|
|
{
|
|
/*
|
|
* Inline data segments are not allowed to cross 64 byte
|
|
@@ -634,7 +634,8 @@ void mlx4_set_sq_sizes(struct mlx4_qp *q
|
|
int wqe_size;
|
|
struct mlx4_context *ctx = to_mctx(qp->ibv_qp.context);
|
|
|
|
- wqe_size = (1 << qp->sq.wqe_shift) - sizeof (struct mlx4_wqe_ctrl_seg);
|
|
+ wqe_size = min((1 << qp->sq.wqe_shift), MLX4_MAX_WQE_SIZE) -
|
|
+ sizeof (struct mlx4_wqe_ctrl_seg);
|
|
switch (type) {
|
|
case IBV_QPT_UD:
|
|
wqe_size -= sizeof (struct mlx4_wqe_datagram_seg);
|