diff --git a/sys/dev/mlx4/device.h b/sys/dev/mlx4/device.h index 6fab35f3e50f..2095df7a700a 100644 --- a/sys/dev/mlx4/device.h +++ b/sys/dev/mlx4/device.h @@ -218,6 +218,23 @@ enum { MLX4_DEV_CAP_CQ_FLAG_IO = 1 << 0 }; +enum { + MLX4_QUERY_FUNC_FLAGS_BF_RES_QP = 1LL << 0 +}; + +/* bit enums for an 8-bit flags field indicating special use + * QPs which require special handling in qp_reserve_range. + * Currently, this only includes QPs used by the ETH interface, + * where we expect to use blueflame. These QPs must not have + * bits 6 and 7 set in their qp number. + * + * This enum may use only bits 0..7. + */ +enum { + MLX4_RESERVE_ETH_BF_QP = 1 << 7, +}; + + enum { MLX4_DEV_CAP_64B_EQE_ENABLED = 1LL << 0, MLX4_DEV_CAP_64B_CQE_ENABLED = 1LL << 1 @@ -533,6 +550,7 @@ struct mlx4_caps { u32 max_basic_counters; u32 max_extended_counters; u8 def_counter_index[MLX4_MAX_PORTS + 1]; + u8 alloc_res_qp_mask; }; struct mlx4_buf_list { diff --git a/sys/dev/mlx4/mlx4_core/fw.h b/sys/dev/mlx4/mlx4_core/fw.h index 5649bc0771a3..748b2841d93d 100644 --- a/sys/dev/mlx4/mlx4_core/fw.h +++ b/sys/dev/mlx4/mlx4_core/fw.h @@ -145,6 +145,7 @@ struct mlx4_func_cap { u8 physical_port; u8 port_flags; u8 def_counter_index; + u8 extra_flags; }; struct mlx4_func { diff --git a/sys/dev/mlx4/mlx4_core/mlx4_fw.c b/sys/dev/mlx4/mlx4_core/mlx4_fw.c index 1a52c599022d..5d137d1637f4 100644 --- a/sys/dev/mlx4/mlx4_core/mlx4_fw.c +++ b/sys/dev/mlx4/mlx4_core/mlx4_fw.c @@ -265,10 +265,15 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, #define QUERY_FUNC_CAP_MTT_QUOTA_OFFSET 0x64 #define QUERY_FUNC_CAP_MCG_QUOTA_OFFSET 0x68 +#define QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET 0x6c + #define QUERY_FUNC_CAP_FMR_FLAG 0x80 #define QUERY_FUNC_CAP_FLAG_RDMA 0x40 #define QUERY_FUNC_CAP_FLAG_ETH 0x80 #define QUERY_FUNC_CAP_FLAG_QUOTAS 0x10 +#define QUERY_FUNC_CAP_FLAG_VALID_MAILBOX 0x04 + +#define QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG (1UL << 31) /* when opcode modifier = 1 */ #define QUERY_FUNC_CAP_PHYS_PORT_OFFSET 0x3 @@ -322,7 +327,7 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, } else if (vhcr->op_modifier == 0) { /* enable rdma and ethernet interfaces, and new quota locations */ field = (QUERY_FUNC_CAP_FLAG_ETH | QUERY_FUNC_CAP_FLAG_RDMA | - QUERY_FUNC_CAP_FLAG_QUOTAS); + QUERY_FUNC_CAP_FLAG_QUOTAS | QUERY_FUNC_CAP_FLAG_VALID_MAILBOX); MLX4_PUT(outbox->buf, field, QUERY_FUNC_CAP_FLAGS_OFFSET); field = dev->caps.num_ports; @@ -382,6 +387,8 @@ int mlx4_QUERY_FUNC_CAP_wrapper(struct mlx4_dev *dev, int slave, MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET); MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_MCG_QUOTA_OFFSET_DEP); + size = QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG; + MLX4_PUT(outbox->buf, size, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); } else err = -EINVAL; @@ -474,6 +481,17 @@ int mlx4_QUERY_FUNC_CAP(struct mlx4_dev *dev, u32 gen_or_port, MLX4_GET(size, outbox, QUERY_FUNC_CAP_RESERVED_EQ_OFFSET); func_cap->reserved_eq = size & 0xFFFFFF; + func_cap->extra_flags = 0; + + /* Mailbox data from 0x6c and onward should only be treated if + * QUERY_FUNC_CAP_FLAG_VALID_MAILBOX is set in func_cap->flags + */ + if (func_cap->flags & QUERY_FUNC_CAP_FLAG_VALID_MAILBOX) { + MLX4_GET(size, outbox, QUERY_FUNC_CAP_EXTRA_FLAGS_OFFSET); + if (size & QUERY_FUNC_CAP_EXTRA_FLAGS_BF_QP_ALLOC_FLAG) + func_cap->extra_flags |= MLX4_QUERY_FUNC_FLAGS_BF_RES_QP; + } + goto out; } diff --git a/sys/dev/mlx4/mlx4_core/mlx4_main.c b/sys/dev/mlx4/mlx4_core/mlx4_main.c index 7389194977e9..9bef78ac50cd 100644 --- a/sys/dev/mlx4/mlx4_core/mlx4_main.c +++ b/sys/dev/mlx4/mlx4_core/mlx4_main.c @@ -849,6 +849,11 @@ static int mlx4_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap) if (!mlx4_is_slave(dev)) { for (i = 0; i < dev->caps.num_ports; ++i) dev->caps.def_counter_index[i] = i << 1; + + dev->caps.alloc_res_qp_mask = + (dev->caps.bf_reg_size ? MLX4_RESERVE_ETH_BF_QP : 0); + } else { + dev->caps.alloc_res_qp_mask = 0; } return 0; @@ -1114,6 +1119,10 @@ static int mlx4_slave_cap(struct mlx4_dev *dev) slave_adjust_steering_mode(dev, &dev_cap, &hca_param); + if (func_cap.extra_flags & MLX4_QUERY_FUNC_FLAGS_BF_RES_QP && + dev->caps.bf_reg_size) + dev->caps.alloc_res_qp_mask |= MLX4_RESERVE_ETH_BF_QP; + return 0; err_mem: diff --git a/sys/dev/mlx4/mlx4_core/mlx4_qp.c b/sys/dev/mlx4/mlx4_core/mlx4_qp.c index 479771997a88..3b40b4563d16 100644 --- a/sys/dev/mlx4/mlx4_core/mlx4_qp.c +++ b/sys/dev/mlx4/mlx4_core/mlx4_qp.c @@ -242,6 +242,9 @@ int mlx4_qp_reserve_range(struct mlx4_dev *dev, int cnt, int align, u64 out_param; int err; + /* Turn off all unsupported QP allocation flags */ + flags &= dev->caps.alloc_res_qp_mask; + if (mlx4_is_mfunc(dev)) { set_param_l(&in_param, (((u32) flags) << 24) | (u32) cnt); set_param_h(&in_param, align); diff --git a/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c b/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c index 821c6dbf1696..43edcac1bfc9 100644 --- a/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c +++ b/sys/dev/mlx4/mlx4_core/mlx4_resource_tracker.c @@ -1544,7 +1544,10 @@ static int qp_alloc_res(struct mlx4_dev *dev, int slave, int op, int cmd, switch (op) { case RES_OP_RESERVE: count = get_param_l(&in_param) & 0xffffff; - flags = get_param_l(&in_param) >> 24; + /* Turn off all unsupported QP allocation flags that the + * slave tries to set. + */ + flags = (get_param_l(&in_param) >> 24) & dev->caps.alloc_res_qp_mask; align = get_param_h(&in_param); err = mlx4_grant_resource(dev, slave, RES_QP, count, 0); if (err)