d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ibcore: Kernel space update based on Linux 5.7-rc1.

Browse Source 
Overview:

This is the first stage of a RDMA stack upgrade introducing kernel
changes only based on Linux 5.7-rc1.

This patch is based on about four main areas of work:
- Update of the IB uobjects system:
  - The memory holding so-called AH, CQ, PD, SRQ and UCONTEXT objects
    is now managed by ibcore. This also require some changes in the
    kernel verbs API. The updated verbs changes are typically about
    initialize and deinitialize objects, and remove allocation and
    free of memory.

- Update of the uverbs IOCTL framework:
  - The parsing and handling of user-space commands has been
    completely refactored to integrate with the updated IB uobjects
    system.

- Various changes and updates to the generic uverbs interfaces in
  device drivers including the new uAPI surface.

- The mlx5_ib_devx.c in mlx5ib and related mlx5 core changes.

Dependencies:

- The mlx4ib driver code has been updated with the minimum changes
needed.

- The mlx5ib driver code has been updated with the minimum changes
needed including DV support.

Compatibility:

- All user-space facing APIs are backwards compatible after this
  change.

- All kernel-space facing RDMA APIs are backwards compatible after
  this change, with exception of ib_create_ah() and ib_destroy_ah()
  which takes a new flag.

- The "ib_device_ops" structure exist, but only contains the driver ID
  and some structure sizes.

Differences from Linux:

- Infiniband drivers must use the INIT_IB_DEVICE_OPS() macro to set
  the sizes needed for allocating various IB objects, when adding
  IB device instances.

Security:

- PRIV_NET_RAW is needed to use raw ethernet transmit features.
- PRIV_DRIVER is needed to use other privileged operations.

Based on upstream Linux, Torvalds (5.7-rc1):
8632e9b5645bbc2331d21d892b0d6961c1a08429

MFC after:	1 week
Reviewed by:	kib
Differential Revision:	https://reviews.freebsd.org/D31149
Sponsored by:	NVIDIA Networking
This commit is contained in:
Hans Petter Selasky 2021-06-16 15:02:00 +02:00
parent 273728b125
commit b633e08c70
95 changed files with 16352 additions and 5097 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
sys/conf/files
View File

@ -4628,6 +4628,8 @@ ofed/drivers/infiniband/core/ib_cm.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_cma.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_core_uverbs.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_cq.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_device.c optional ofed \
@ -4648,6 +4650,8 @@ ofed/drivers/infiniband/core/ib_multicast.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_packer.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_rdma_core.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_roce_gid_mgmt.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_sa_query.c optional ofed \
@ -4668,10 +4672,30 @@ ofed/drivers/infiniband/core/ib_user_mad.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_cmd.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_ioctl.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_main.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_marshall.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_async_fd.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_counters.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_cq.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_device.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_dm.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_flow_action.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_std_types_mr.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_uverbs_uapi.c optional ofed \
compile-with "${OFED_C}"
ofed/drivers/infiniband/core/ib_verbs.c optional ofed \
compile-with "${OFED_C}"
@ -4827,6 +4851,8 @@ dev/mlx5/mlx5_ib/mlx5_ib_cong.c optional mlx5ib pci ofed \
compile-with "${OFED_C}"
dev/mlx5/mlx5_ib/mlx5_ib_cq.c optional mlx5ib pci ofed \
compile-with "${OFED_C}"
dev/mlx5/mlx5_ib/mlx5_ib_devx.c optional mlx5ib pci ofed \
compile-with "${OFED_C}"
dev/mlx5/mlx5_ib/mlx5_ib_doorbell.c optional mlx5ib pci ofed \
compile-with "${OFED_C}"
dev/mlx5/mlx5_ib/mlx5_ib_gsi.c optional mlx5ib pci ofed \

32
sys/dev/cxgbe/iw_cxgbe/cq.c
View File

@ -883,7 +883,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
return !err || err == -ENODATA ? npolled : err;
}
int c4iw_destroy_cq(struct ib_cq *ib_cq)
void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata)
{
struct c4iw_cq *chp;
struct c4iw_ucontext *ucontext;
@ -895,22 +895,20 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq)
atomic_dec(&chp->refcnt);
wait_event(chp->wait, !atomic_read(&chp->refcnt));
ucontext = ib_cq->uobject ? to_c4iw_ucontext(ib_cq->uobject->context)
: NULL;
ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,
ibucontext);
destroy_cq(&chp->rhp->rdev, &chp->cq,
ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx);
kfree(chp);
return 0;
}
struct ib_cq *
c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context, struct ib_udata *udata)
int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
struct c4iw_dev *rhp;
struct c4iw_cq *chp;
struct c4iw_cq *chp = to_c4iw_cq(ibcq);
struct c4iw_create_cq_resp uresp;
struct c4iw_ucontext *ucontext = NULL;
int ret;
@ -919,17 +917,12 @@ c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr,
CTR3(KTR_IW_CXGBE, "%s ib_dev %p entries %d", __func__, ibdev, entries);
if (attr->flags)
return ERR_PTR(-EINVAL);
return -EINVAL;
rhp = to_c4iw_dev(ibdev);
chp = kzalloc(sizeof(*chp), GFP_KERNEL);
if (!chp)
return ERR_PTR(-ENOMEM);
if (ib_context)
ucontext = to_c4iw_ucontext(ib_context);
ucontext = rdma_udata_to_drv_context(udata, struct c4iw_ucontext,
ibucontext);
/* account for the status page. */
entries++;
@ -1020,7 +1013,7 @@ c4iw_create_cq(struct ib_device *ibdev, const struct ib_cq_init_attr *attr,
"%s cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx",
__func__, chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize,
(unsigned long long) chp->cq.dma_addr);
return &chp->ibcq;
return 0;
err5:
kfree(mm2);
err4:
@ -1031,8 +1024,7 @@ err2:
destroy_cq(&chp->rhp->rdev, &chp->cq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
err1:
kfree(chp);
return ERR_PTR(ret);
return ret;
}
int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)

36
sys/dev/cxgbe/iw_cxgbe/iw_cxgbe.h
View File

@ -55,6 +55,7 @@
#include <rdma/ib_verbs.h>
#include <rdma/iw_cm.h>
#include <rdma/uverbs_ioctl.h>
#include "common/common.h"
#include "common/t4_msg.h"
@ -474,6 +475,14 @@ struct c4iw_qp_attributes {
u8 send_term;
};
struct c4iw_ib_srq {
struct ib_srq ibsrq;
};
struct c4iw_ib_ah {
struct ib_ah ibah;
};
struct c4iw_qp {
struct ib_qp ibqp;
struct c4iw_dev *rhp;
@ -501,7 +510,6 @@ struct c4iw_ucontext {
u32 key;
spinlock_t mmap_lock;
struct list_head mmaps;
struct kref kref;
};
static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
@ -509,17 +517,6 @@ static inline struct c4iw_ucontext *to_c4iw_ucontext(struct ib_ucontext *c)
return container_of(c, struct c4iw_ucontext, ibucontext);
}
void _c4iw_free_ucontext(struct kref *kref);
static inline void c4iw_put_ucontext(struct c4iw_ucontext *ucontext)
{
kref_put(&ucontext->kref, _c4iw_free_ucontext);
}
static inline void c4iw_get_ucontext(struct c4iw_ucontext *ucontext)
{
kref_get(&ucontext->kref);
}
struct c4iw_mm_entry {
struct list_head entry;
u64 addr;
@ -938,7 +935,7 @@ int c4iw_reject_cr(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len);
void c4iw_qp_add_ref(struct ib_qp *qp);
void c4iw_qp_rem_ref(struct ib_qp *qp);
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg);
u32 max_num_sg, struct ib_udata *udata);
int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
int c4iw_dealloc_mw(struct ib_mw *mw);
@ -947,16 +944,15 @@ struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, u64
virt, int acc, struct ib_udata *udata);
struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc);
int c4iw_dereg_mr(struct ib_mr *ib_mr);
int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata);
void c4iw_invalidate_mr(struct c4iw_dev *rhp, u32 rkey);
int c4iw_destroy_cq(struct ib_cq *ib_cq);
struct ib_cq *c4iw_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_context,
struct ib_udata *udata);
void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata);
int c4iw_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
int c4iw_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int c4iw_destroy_qp(struct ib_qp *ib_qp);
int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata);
struct ib_qp *c4iw_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *attrs,
struct ib_udata *udata);

4
sys/dev/cxgbe/iw_cxgbe/mem.c
View File

@ -610,7 +610,7 @@ int c4iw_dealloc_mw(struct ib_mw *mw)
struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg)
u32 max_num_sg, struct ib_udata *udata)
{
struct c4iw_dev *rhp;
struct c4iw_pd *php;
@ -700,7 +700,7 @@ int c4iw_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
}
int c4iw_dereg_mr(struct ib_mr *ib_mr)
int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
{
struct c4iw_dev *rhp;
struct c4iw_mr *mhp;

100
sys/dev/cxgbe/iw_cxgbe/provider.c
View File

@ -58,18 +58,17 @@ static int c4iw_modify_port(struct ib_device *ibdev,
return -ENOSYS;
}
static struct ib_ah *c4iw_ah_create(struct ib_pd *pd,
struct ib_ah_attr *ah_attr,
struct ib_udata *udata)
{
return ERR_PTR(-ENOSYS);
}
static int c4iw_ah_destroy(struct ib_ah *ah)
static int c4iw_ah_create(struct ib_ah *ah,
struct ib_ah_attr *ah_attr, u32 flags,
struct ib_udata *udata)
{
return -ENOSYS;
}
static void c4iw_ah_destroy(struct ib_ah *ah, u32 flags)
{
}
static int c4iw_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
{
return -ENOSYS;
@ -93,35 +92,27 @@ static int c4iw_process_mad(struct ib_device *ibdev, int mad_flags,
return -ENOSYS;
}
void _c4iw_free_ucontext(struct kref *kref)
static void c4iw_dealloc_ucontext(struct ib_ucontext *context)
{
struct c4iw_ucontext *ucontext;
struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
struct c4iw_dev *rhp;
struct c4iw_mm_entry *mm, *tmp;
ucontext = container_of(kref, struct c4iw_ucontext, kref);
pr_debug("context %p\n", context);
rhp = to_c4iw_dev(ucontext->ibucontext.device);
CTR2(KTR_IW_CXGBE, "%s ucontext %p", __func__, ucontext);
list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
kfree(mm);
c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx);
kfree(ucontext);
}
static int c4iw_dealloc_ucontext(struct ib_ucontext *context)
static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
struct ib_udata *udata)
{
struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
CTR2(KTR_IW_CXGBE, "%s context %p", __func__, context);
c4iw_put_ucontext(ucontext);
return 0;
}
static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata)
{
struct c4iw_ucontext *context;
struct ib_device *ibdev = ucontext->device;
struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);
struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
static int warned;
struct c4iw_alloc_ucontext_resp uresp;
@ -129,16 +120,9 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev,
struct c4iw_mm_entry *mm = NULL;
PDBG("%s ibdev %p\n", __func__, ibdev);
context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context) {
ret = -ENOMEM;
goto err;
}
c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx);
INIT_LIST_HEAD(&context->mmaps);
spin_lock_init(&context->mmap_lock);
kref_init(&context->kref);
if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) {
if (!warned++)
@ -150,7 +134,7 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev,
mm = kmalloc(sizeof *mm, GFP_KERNEL);
if (!mm)
goto err_free;
goto err;
uresp.status_page_size = PAGE_SIZE;
@ -169,13 +153,11 @@ static struct ib_ucontext *c4iw_alloc_ucontext(struct ib_device *ibdev,
mm->len = PAGE_SIZE;
insert_mmap(context, mm);
}
return &context->ibucontext;
return 0;
err_mm:
kfree(mm);
err_free:
kfree(context);
err:
return ERR_PTR(ret);
return ret;
}
static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
@ -226,8 +208,8 @@ static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
return ret;
}
static int
c4iw_deallocate_pd(struct ib_pd *pd)
static void
c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)
{
struct c4iw_pd *php = to_c4iw_pd(pd);
struct c4iw_dev *rhp = php->rhp;
@ -238,36 +220,29 @@ c4iw_deallocate_pd(struct ib_pd *pd)
mutex_lock(&rhp->rdev.stats.lock);
rhp->rdev.stats.pd.cur--;
mutex_unlock(&rhp->rdev.stats.lock);
kfree(php);
return (0);
}
static struct ib_pd *
c4iw_allocate_pd(struct ib_device *ibdev, struct ib_ucontext *context,
struct ib_udata *udata)
static int
c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
{
struct c4iw_pd *php;
struct c4iw_pd *php = to_c4iw_pd(pd);
struct ib_device *ibdev = pd->device;
u32 pdid;
struct c4iw_dev *rhp;
CTR4(KTR_IW_CXGBE, "%s: ibdev %p, context %p, data %p", __func__, ibdev,
context, udata);
CTR4(KTR_IW_CXGBE, "%s: ibdev %p, pd %p, data %p", __func__, ibdev,
pd, udata);
rhp = (struct c4iw_dev *) ibdev;
pdid = c4iw_get_resource(&rhp->rdev.resource.pdid_table);
if (!pdid)
return ERR_PTR(-EINVAL);
php = kzalloc(sizeof(*php), GFP_KERNEL);
if (!php) {
c4iw_put_resource(&rhp->rdev.resource.pdid_table, pdid);
return ERR_PTR(-ENOMEM);
}
return -EINVAL;
php->pdid = pdid;
php->rhp = rhp;
if (context) {
if (udata) {
if (ib_copy_to_udata(udata, &php->pdid, sizeof(u32))) {
c4iw_deallocate_pd(&php->ibpd);
return ERR_PTR(-EFAULT);
c4iw_deallocate_pd(&php->ibpd, udata);
return -EFAULT;
}
}
mutex_lock(&rhp->rdev.stats.lock);
@ -276,10 +251,10 @@ c4iw_allocate_pd(struct ib_device *ibdev, struct ib_ucontext *context,
rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur;
mutex_unlock(&rhp->rdev.stats.lock);
CTR6(KTR_IW_CXGBE,
CTR5(KTR_IW_CXGBE,
"%s: ibdev %p, context %p, data %p, pddid 0x%x, pd %p", __func__,
ibdev, context, udata, pdid, php);
return (&php->ibpd);
ibdev, udata, pdid, php);
return (0);
}
static int
@ -436,6 +411,13 @@ c4iw_register_device(struct c4iw_dev *dev)
ret = linux_pci_attach_device(sc->dev, NULL, NULL, &dev->pdev);
if (ret)
return (ret);
#define c4iw_ib_cq c4iw_cq
#define c4iw_ib_pd c4iw_pd
#define c4iw_ib_qp c4iw_qp
#define c4iw_ib_ucontext c4iw_ucontext
INIT_IB_DEVICE_OPS(&ibdev->ops, c4iw, CXGB4);
strlcpy(ibdev->name, device_get_nameunit(sc->dev), sizeof(ibdev->name));
memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid));
memcpy(&ibdev->node_guid, sc->port[0]->vi[0].hw_addr, ETHER_ADDR_LEN);

5
sys/dev/cxgbe/iw_cxgbe/qp.c
View File

@ -583,8 +583,6 @@ static void free_qp_work(struct work_struct *work)
destroy_qp(&rhp->rdev, &qhp->wq,
ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
if (ucontext)
c4iw_put_ucontext(ucontext);
kfree(qhp);
}
@ -1678,7 +1676,7 @@ out:
return ret;
}
int c4iw_destroy_qp(struct ib_qp *ib_qp)
int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata)
{
struct c4iw_dev *rhp;
struct c4iw_qp *qhp;
@ -1879,7 +1877,6 @@ c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs,
rq_db_key_mm->len);
insert_mmap(ucontext, rq_db_key_mm);
c4iw_get_ucontext(ucontext);
qhp->ucontext = ucontext;
}
qhp->ibqp.qp_num = qhp->wq.sq.qid;

37
sys/dev/mlx4/mlx4_ib/mlx4_ib.h
View File

@ -83,16 +83,11 @@ enum hw_bar_type {
HW_BAR_COUNT
};
struct mlx4_ib_vma_private_data {
struct vm_area_struct *vma;
};
struct mlx4_ib_ucontext {
struct ib_ucontext ibucontext;
struct mlx4_uar uar;
struct list_head db_page_list;
struct mutex db_page_mutex;
struct mlx4_ib_vma_private_data hw_bar_info[HW_BAR_COUNT];
};
struct mlx4_ib_pd {
@ -726,39 +721,37 @@ int mlx4_ib_umem_write_mtt(struct mlx4_ib_dev *dev, struct mlx4_mtt *mtt,
struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
u64 virt_addr, int access_flags,
struct ib_udata *udata);
int mlx4_ib_dereg_mr(struct ib_mr *mr);
int mlx4_ib_dereg_mr(struct ib_mr *mr, struct ib_udata *udata);
struct ib_mw *mlx4_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata);
int mlx4_ib_dealloc_mw(struct ib_mw *mw);
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg);
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata);
struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata);
int mlx4_ib_destroy_cq(struct ib_cq *cq);
int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags);
void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
void mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq);
struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
struct ib_udata *udata);
int mlx4_ib_create_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr, u32 flags,
struct ib_udata *udata);
int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct ib_ah_attr *ah_attr,
int slave_sgid_index, u8 *s_mac, u16 vlan_tag);
int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
int mlx4_ib_destroy_ah(struct ib_ah *ah);
void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags);
struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata);
int mlx4_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
struct ib_udata *udata);
int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
int mlx4_ib_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
int mlx4_ib_destroy_srq(struct ib_srq *srq);
void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index);
int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
const struct ib_recv_wr **bad_wr);
@ -766,7 +759,7 @@ int mlx4_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata);
int mlx4_ib_destroy_qp(struct ib_qp *qp);
int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata);
int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,

74
sys/dev/mlx4/mlx4_ib/mlx4_ib_ah.c
View File

@ -42,10 +42,11 @@
#include "mlx4_ib.h"
static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
struct mlx4_ib_ah *ah)
static int create_ib_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
{
struct mlx4_dev *dev = to_mdev(pd->device)->dev;
struct ib_pd *pd = ib_ah->pd;
struct mlx4_ib_ah *ah = to_mah(ib_ah);
struct mlx4_dev *dev = to_mdev(ib_ah->device)->dev;
ah->av.ib.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24));
ah->av.ib.g_slid = ah_attr->src_path_bits;
@ -67,14 +68,14 @@ static struct ib_ah *create_ib_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
!(1 << ah->av.ib.stat_rate & dev->caps.stat_rate_support))
--ah->av.ib.stat_rate;
}
return &ah->ibah;
return 0;
}
static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
struct mlx4_ib_ah *ah)
static int create_iboe_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr)
{
struct mlx4_ib_dev *ibdev = to_mdev(pd->device);
struct ib_pd *pd = ib_ah->pd;
struct mlx4_ib_dev *ibdev = to_mdev(ib_ah->device);
struct mlx4_ib_ah *ah = to_mah(ib_ah);
struct mlx4_dev *dev = ibdev->dev;
int is_mcast = 0;
struct in6_addr in6;
@ -93,7 +94,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
ret = ib_get_cached_gid(pd->device, ah_attr->port_num,
ah_attr->grh.sgid_index, &sgid, &gid_attr);
if (ret)
return ERR_PTR(ret);
return ret;
eth_zero_addr(ah->av.eth.s_mac);
if (gid_attr.ndev) {
vlan_tag = rdma_vlan_dev_vlan_id(gid_attr.ndev);
@ -105,7 +106,7 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
ah->av.eth.port_pd = cpu_to_be32(to_mpd(pd)->pdn | (ah_attr->port_num << 24));
ret = mlx4_ib_gid_index_to_real_index(ibdev, ah_attr->port_num, ah_attr->grh.sgid_index);
if (ret < 0)
return ERR_PTR(ret);
return ret;
ah->av.eth.gid_index = ret;
ah->av.eth.vlan = cpu_to_be16(vlan_tag);
ah->av.eth.hop_limit = ah_attr->grh.hop_limit;
@ -125,23 +126,15 @@ static struct ib_ah *create_iboe_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr
memcpy(ah->av.eth.dgid, ah_attr->grh.dgid.raw, 16);
ah->av.eth.sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 29);
return &ah->ibah;
return 0;
}
struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
struct ib_udata *udata)
int mlx4_ib_create_ah(struct ib_ah *ib_ah, struct ib_ah_attr *ah_attr,
u32 flags, struct ib_udata *udata)
{
struct mlx4_ib_ah *ah;
struct ib_ah *ret;
ah = kzalloc(sizeof *ah, GFP_ATOMIC);
if (!ah)
return ERR_PTR(-ENOMEM);
if (rdma_port_get_link_layer(pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET) {
if (rdma_port_get_link_layer(ib_ah->pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET) {
if (!(ah_attr->ah_flags & IB_AH_GRH)) {
ret = ERR_PTR(-EINVAL);
return -EINVAL;
} else {
/*
* TBD: need to handle the case when we get
@ -151,15 +144,35 @@ struct ib_ah *mlx4_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
* local addresses which we can translate
* without going to sleep.
*/
ret = create_iboe_ah(pd, ah_attr, ah);
return create_iboe_ah(ib_ah, ah_attr);
}
}
return create_ib_ah(ib_ah, ah_attr);
}
if (IS_ERR(ret))
kfree(ah);
int mlx4_ib_create_ah_slave(struct ib_ah *ah, struct ib_ah_attr *ah_attr,
int slave_sgid_index, u8 *s_mac, u16 vlan_tag)
{
struct ib_ah_attr slave_attr = *ah_attr;
struct mlx4_ib_ah *mah = to_mah(ah);
int ret;
slave_attr.grh.sgid_index = slave_sgid_index;
ret = mlx4_ib_create_ah(ah, &slave_attr, 0, NULL);
if (ret)
return ret;
} else
return create_ib_ah(pd, ah_attr, ah); /* never fails */
/* get rid of force-loopback bit */
mah->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
if (rdma_port_get_link_layer(ah->pd->device, ah_attr->port_num) == IB_LINK_LAYER_ETHERNET)
memcpy(mah->av.eth.s_mac, s_mac, 6);
if (vlan_tag < 0x1000)
vlan_tag |= (ah_attr->sl & 7) << 13;
mah->av.eth.vlan = cpu_to_be16(vlan_tag);
return 0;
}
int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
@ -195,8 +208,7 @@ int mlx4_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
return 0;
}
int mlx4_ib_destroy_ah(struct ib_ah *ah)
void mlx4_ib_destroy_ah(struct ib_ah *ah, u32 flags)
{
kfree(to_mah(ah));
return 0;
return;
}

89
sys/dev/mlx4/mlx4_ib/mlx4_ib_cq.c
View File

@ -39,6 +39,7 @@
#include "mlx4_ib.h"
#include <rdma/mlx4-abi.h>
#include <rdma/uverbs_ioctl.h>
static void mlx4_ib_cq_comp(struct mlx4_cq *cq)
{
@ -135,14 +136,16 @@ static void mlx4_ib_free_cq_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq_buf *
mlx4_buf_free(dev->dev, (cqe + 1) * buf->entry_size, &buf->buf);
}
static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_ucontext *context,
struct mlx4_ib_cq_buf *buf, struct ib_umem **umem,
u64 buf_addr, int cqe)
static int mlx4_ib_get_cq_umem(struct mlx4_ib_dev *dev, struct ib_udata *udata,
struct mlx4_ib_cq_buf *buf,
struct ib_umem **umem, u64 buf_addr, int cqe)
{
int err;
int cqe_size = dev->dev->caps.cqe_size;
struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx4_ib_ucontext, ibucontext);
*umem = ib_umem_get(context, buf_addr, cqe * cqe_size,
*umem = ib_umem_get(&context->ibucontext, buf_addr, cqe * cqe_size,
IB_ACCESS_LOCAL_WRITE, 1);
if (IS_ERR(*umem))
return PTR_ERR(*umem);
@ -168,27 +171,25 @@ err_buf:
}
#define CQ_CREATE_FLAGS_SUPPORTED IB_CQ_FLAGS_TIMESTAMP_COMPLETION
struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)
int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx4_ib_dev *dev = to_mdev(ibdev);
struct mlx4_ib_cq *cq;
struct mlx4_ib_cq *cq = to_mcq(ibcq);
struct mlx4_uar *uar;
void *buf_addr;
int err;
struct mlx4_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx4_ib_ucontext, ibucontext);
if (entries < 1 || entries > dev->dev->caps.max_cqes)
return ERR_PTR(-EINVAL);
return -EINVAL;
if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED)
return ERR_PTR(-EINVAL);
cq = kmalloc(sizeof *cq, GFP_KERNEL);
if (!cq)
return ERR_PTR(-ENOMEM);
return -EINVAL;
entries = roundup_pow_of_two(entries + 1);
cq->ibcq.cqe = entries - 1;
@ -200,7 +201,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
INIT_LIST_HEAD(&cq->send_qp_list);
INIT_LIST_HEAD(&cq->recv_qp_list);
if (context) {
if (udata) {
struct mlx4_ib_create_cq ucmd;
if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
@ -208,17 +209,17 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
goto err_cq;
}
err = mlx4_ib_get_cq_umem(dev, context, &cq->buf, &cq->umem,
buf_addr = (void *)(unsigned long)ucmd.buf_addr;
err = mlx4_ib_get_cq_umem(dev, udata, &cq->buf, &cq->umem,
ucmd.buf_addr, entries);
if (err)
goto err_cq;
err = mlx4_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
&cq->db);
err = mlx4_ib_db_map_user(context, ucmd.db_addr, &cq->db);
if (err)
goto err_mtt;
uar = &to_mucontext(context)->uar;
uar = &context->uar;
} else {
err = mlx4_db_alloc(dev->dev, &cq->db, 1, GFP_KERNEL);
if (err)
@ -233,6 +234,8 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
if (err)
goto err_db;
buf_addr = &cq->buf.buf;
uar = &dev->priv_uar;
}
@ -248,37 +251,33 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev,
cq->mcq.comp = mlx4_ib_cq_comp;
cq->mcq.event = mlx4_ib_cq_event;
if (context)
if (udata)
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof (__u32))) {
err = -EFAULT;
goto err_cq_free;
}
return &cq->ibcq;
return 0;
err_cq_free:
mlx4_cq_free(dev->dev, &cq->mcq);
err_dbmap:
if (context)
mlx4_ib_db_unmap_user(to_mucontext(context), &cq->db);
if (udata)
mlx4_ib_db_unmap_user(context, &cq->db);
err_mtt:
mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt);
if (context)
ib_umem_release(cq->umem);
else
ib_umem_release(cq->umem);
if (!udata)
mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe);
err_db:
if (!context)
if (!udata)
mlx4_db_free(dev->dev, &cq->db);
err_cq:
kfree(cq);
return ERR_PTR(err);
return err;
}
static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq,
@ -321,7 +320,7 @@ static int mlx4_alloc_resize_umem(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq
if (!cq->resize_buf)
return -ENOMEM;
err = mlx4_ib_get_cq_umem(dev, cq->umem->context, &cq->resize_buf->buf,
err = mlx4_ib_get_cq_umem(dev, udata, &cq->resize_buf->buf,
&cq->resize_umem, ucmd.buf_addr, entries);
if (err) {
kfree(cq->resize_buf);
@ -460,18 +459,15 @@ err_buf:
kfree(cq->resize_buf);
cq->resize_buf = NULL;
if (cq->resize_umem) {
ib_umem_release(cq->resize_umem);
cq->resize_umem = NULL;
}
ib_umem_release(cq->resize_umem);
cq->resize_umem = NULL;
out:
mutex_unlock(&cq->resize_mutex);
return err;
}
int mlx4_ib_destroy_cq(struct ib_cq *cq)
void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(cq->device);
struct mlx4_ib_cq *mcq = to_mcq(cq);
@ -479,17 +475,18 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq)
mlx4_cq_free(dev->dev, &mcq->mcq);
mlx4_mtt_cleanup(dev->dev, &mcq->buf.mtt);
if (cq->uobject) {
mlx4_ib_db_unmap_user(to_mucontext(cq->uobject->context), &mcq->db);
ib_umem_release(mcq->umem);
if (udata) {
mlx4_ib_db_unmap_user(
rdma_udata_to_drv_context(
udata,
struct mlx4_ib_ucontext,
ibucontext),
&mcq->db);
} else {
mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe);
mlx4_db_free(dev->dev, &mcq->db);
}
kfree(mcq);
return 0;
ib_umem_release(mcq->umem);
}
static void dump_cqe(void *cqe)

65
sys/dev/mlx4/mlx4_ib/mlx4_ib_mad.c
View File

@ -199,13 +199,13 @@ static void update_sm_ah(struct mlx4_ib_dev *dev, u8 port_num, u16 lid, u8 sl)
ah_attr.port_num = port_num;
new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd,
&ah_attr);
&ah_attr, 0);
if (IS_ERR(new_ah))
return;
spin_lock_irqsave(&dev->sm_lock, flags);
if (dev->sm_ah[port_num - 1])
ib_destroy_ah(dev->sm_ah[port_num - 1]);
ib_destroy_ah(dev->sm_ah[port_num - 1], 0);
dev->sm_ah[port_num - 1] = new_ah;
spin_unlock_irqrestore(&dev->sm_lock, flags);
}
@ -540,7 +540,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
memcpy(&attr.grh.dgid.raw[0], &grh->dgid.raw[0], 16);
attr.ah_flags = IB_AH_GRH;
}
ah = ib_create_ah(tun_ctx->pd, &attr);
ah = ib_create_ah(tun_ctx->pd, &attr, 0);
if (IS_ERR(ah))
return -ENOMEM;
@ -557,7 +557,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
tun_mad = (struct mlx4_rcv_tunnel_mad *) (tun_qp->tx_ring[tun_tx_ix].buf.addr);
if (tun_qp->tx_ring[tun_tx_ix].ah)
ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah);
ib_destroy_ah(tun_qp->tx_ring[tun_tx_ix].ah, 0);
tun_qp->tx_ring[tun_tx_ix].ah = ah;
ib_dma_sync_single_for_cpu(&dev->ib_dev,
tun_qp->tx_ring[tun_tx_ix].buf.map,
@ -632,7 +632,7 @@ int mlx4_ib_send_to_slave(struct mlx4_ib_dev *dev, int slave, u8 port,
spin_unlock(&tun_qp->tx_lock);
tun_qp->tx_ring[tun_tx_ix].ah = NULL;
end:
ib_destroy_ah(ah);
ib_destroy_ah(ah, 0);
return ret;
}
@ -986,7 +986,7 @@ static void send_handler(struct ib_mad_agent *agent,
struct ib_mad_send_wc *mad_send_wc)
{
if (mad_send_wc->send_buf->context[0])
ib_destroy_ah(mad_send_wc->send_buf->context[0]);
ib_destroy_ah(mad_send_wc->send_buf->context[0], 0);
ib_free_send_mad(mad_send_wc->send_buf);
}
@ -1041,7 +1041,7 @@ void mlx4_ib_mad_cleanup(struct mlx4_ib_dev *dev)
}
if (dev->sm_ah[p])
ib_destroy_ah(dev->sm_ah[p]);
ib_destroy_ah(dev->sm_ah[p], 0);
}
}
@ -1334,11 +1334,9 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
struct ib_ah *ah;
struct ib_qp *send_qp = NULL;
unsigned wire_tx_ix = 0;
int ret = 0;
u16 wire_pkey_ix;
int src_qpnum;
u8 sgid_index;
int ret;
sqp_ctx = dev->sriov.sqps[port-1];
@ -1358,16 +1356,20 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
send_qp = sqp->qp;
/* create ah */
sgid_index = attr->grh.sgid_index;
attr->grh.sgid_index = 0;
ah = ib_create_ah(sqp_ctx->pd, attr);
if (IS_ERR(ah))
ah = rdma_zalloc_drv_obj(sqp_ctx->pd->device, ib_ah);
if (!ah)
return -ENOMEM;
attr->grh.sgid_index = sgid_index;
to_mah(ah)->av.ib.gid_index = sgid_index;
/* get rid of force-loopback bit */
to_mah(ah)->av.ib.port_pd &= cpu_to_be32(0x7FFFFFFF);
ah->device = sqp_ctx->pd->device;
ah->pd = sqp_ctx->pd;
/* create ah */
ret = mlx4_ib_create_ah_slave(ah, attr,
attr->grh.sgid_index,
s_mac, vlan_id);
if (ret)
goto out;
spin_lock(&sqp->tx_lock);
if (sqp->tx_ix_head - sqp->tx_ix_tail >=
(MLX4_NUM_TUNNEL_BUFS - 1))
@ -1379,8 +1381,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
goto out;
sqp_mad = (struct mlx4_mad_snd_buf *) (sqp->tx_ring[wire_tx_ix].buf.addr);
if (sqp->tx_ring[wire_tx_ix].ah)
ib_destroy_ah(sqp->tx_ring[wire_tx_ix].ah);
kfree(sqp->tx_ring[wire_tx_ix].ah);
sqp->tx_ring[wire_tx_ix].ah = ah;
ib_dma_sync_single_for_cpu(&dev->ib_dev,
sqp->tx_ring[wire_tx_ix].buf.map,
@ -1409,12 +1410,6 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
wr.wr.num_sge = 1;
wr.wr.opcode = IB_WR_SEND;
wr.wr.send_flags = IB_SEND_SIGNALED;
if (s_mac)
memcpy(to_mah(ah)->av.eth.s_mac, s_mac, 6);
if (vlan_id < 0x1000)
vlan_id |= (attr->sl & 7) << 13;
to_mah(ah)->av.eth.vlan = cpu_to_be16(vlan_id);
ret = ib_post_send(send_qp, &wr.wr, &bad_wr);
if (!ret)
@ -1425,7 +1420,7 @@ int mlx4_ib_send_to_wire(struct mlx4_ib_dev *dev, int slave, u8 port,
spin_unlock(&sqp->tx_lock);
sqp->tx_ring[wire_tx_ix].ah = NULL;
out:
ib_destroy_ah(ah);
kfree(ah);
return ret;
}
@ -1684,7 +1679,7 @@ static void mlx4_ib_free_pv_qp_bufs(struct mlx4_ib_demux_pv_ctx *ctx,
tx_buf_size, DMA_TO_DEVICE);
kfree(tun_qp->tx_ring[i].buf.addr);
if (tun_qp->tx_ring[i].ah)
ib_destroy_ah(tun_qp->tx_ring[i].ah);
ib_destroy_ah(tun_qp->tx_ring[i].ah, 0);
}
kfree(tun_qp->tx_ring);
kfree(tun_qp->ring);
@ -1717,7 +1712,7 @@ static void mlx4_ib_tunnel_comp_worker(struct work_struct *work)
"wrid=0x%llx, status=0x%x\n",
(unsigned long long)wc.wr_id, wc.status);
ib_destroy_ah(tun_qp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
(MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0);
tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
= NULL;
spin_lock(&tun_qp->tx_lock);
@ -1734,7 +1729,7 @@ static void mlx4_ib_tunnel_comp_worker(struct work_struct *work)
ctx->slave, wc.status, (unsigned long long)wc.wr_id);
if (!MLX4_TUN_IS_RECV(wc.wr_id)) {
ib_destroy_ah(tun_qp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
(MLX4_NUM_TUNNEL_BUFS - 1)].ah, 0);
tun_qp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
= NULL;
spin_lock(&tun_qp->tx_lock);
@ -1872,8 +1867,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)
if (wc.status == IB_WC_SUCCESS) {
switch (wc.opcode) {
case IB_WC_SEND:
ib_destroy_ah(sqp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
kfree(sqp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
= NULL;
spin_lock(&sqp->tx_lock);
@ -1902,8 +1897,8 @@ static void mlx4_ib_sqp_comp_worker(struct work_struct *work)
" status = %d, wrid = 0x%llx\n",
ctx->slave, wc.status, (unsigned long long)wc.wr_id);
if (!MLX4_TUN_IS_RECV(wc.wr_id)) {
ib_destroy_ah(sqp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
kfree(sqp->tx_ring[wc.wr_id &
(MLX4_NUM_TUNNEL_BUFS - 1)].ah);
sqp->tx_ring[wc.wr_id & (MLX4_NUM_TUNNEL_BUFS - 1)].ah
= NULL;
spin_lock(&sqp->tx_lock);

197
sys/dev/mlx4/mlx4_ib/mlx4_ib_main.c
View File

@ -1039,17 +1039,18 @@ out:
return err;
}
static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata)
static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata)
{
struct ib_device *ibdev = uctx->device;
struct mlx4_ib_dev *dev = to_mdev(ibdev);
struct mlx4_ib_ucontext *context;
struct mlx4_ib_ucontext *context = to_mucontext(uctx);
struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
struct mlx4_ib_alloc_ucontext_resp resp;
int err;
if (!dev->ib_active)
return ERR_PTR(-EAGAIN);
return -EAGAIN;
if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
resp_v3.qp_tab_size = dev->dev->caps.num_qps;
@ -1063,15 +1064,9 @@ static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
resp.cqe_size = dev->dev->caps.cqe_size;
}
context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context)
return ERR_PTR(-ENOMEM);
err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);
if (err) {
kfree(context);
return ERR_PTR(err);
}
if (err)
return err;
INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex);
@ -1083,176 +1078,87 @@ static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
if (err) {
mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
kfree(context);
return ERR_PTR(-EFAULT);
return -EFAULT;
}
return &context->ibucontext;
return err;
}
static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
static void mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
{
struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);
mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);
kfree(context);
return 0;
}
static void mlx4_ib_vma_open(struct vm_area_struct *area)
{
/* vma_open is called when a new VMA is created on top of our VMA.
* This is done through either mremap flow or split_vma (usually due
* to mlock, madvise, munmap, etc.). We do not support a clone of the
* vma, as this VMA is strongly hardware related. Therefore we set the
* vm_ops of the newly created/cloned VMA to NULL, to prevent it from
* calling us again and trying to do incorrect actions. We assume that
* the original vma size is exactly a single page that there will be no
* "splitting" operations on.
*/
area->vm_ops = NULL;
}
static void mlx4_ib_vma_close(struct vm_area_struct *area)
{
struct mlx4_ib_vma_private_data *mlx4_ib_vma_priv_data;
/* It's guaranteed that all VMAs opened on a FD are closed before the
* file itself is closed, therefore no sync is needed with the regular
* closing flow. (e.g. mlx4_ib_dealloc_ucontext) However need a sync
* with accessing the vma as part of mlx4_ib_disassociate_ucontext.
* The close operation is usually called under mm->mmap_sem except when
* process is exiting. The exiting case is handled explicitly as part
* of mlx4_ib_disassociate_ucontext.
*/
mlx4_ib_vma_priv_data = (struct mlx4_ib_vma_private_data *)
area->vm_private_data;
/* set the vma context pointer to null in the mlx4_ib driver's private
* data to protect against a race condition in mlx4_ib_dissassociate_ucontext().
*/
mlx4_ib_vma_priv_data->vma = NULL;
}
static const struct vm_operations_struct mlx4_ib_vm_ops = {
.open = mlx4_ib_vma_open,
.close = mlx4_ib_vma_close
};
static void mlx4_ib_set_vma_data(struct vm_area_struct *vma,
struct mlx4_ib_vma_private_data *vma_private_data)
{
vma_private_data->vma = vma;
vma->vm_private_data = vma_private_data;
vma->vm_ops = &mlx4_ib_vm_ops;
}
static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
struct mlx4_ib_dev *dev = to_mdev(context->device);
struct mlx4_ib_ucontext *mucontext = to_mucontext(context);
if (vma->vm_end - vma->vm_start != PAGE_SIZE)
return -EINVAL;
switch (vma->vm_pgoff) {
case 0:
return rdma_user_mmap_io(context, vma,
to_mucontext(context)->uar.pfn,
PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot),
NULL);
if (vma->vm_pgoff == 0) {
/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_DB].vma)
case 1:
if (dev->dev->caps.bf_reg_size == 0)
return -EINVAL;
return rdma_user_mmap_io(
context, vma,
to_mucontext(context)->uar.pfn +
dev->dev->caps.num_uars,
PAGE_SIZE, pgprot_writecombine(vma->vm_page_prot),
NULL);
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (io_remap_pfn_range(vma, vma->vm_start,
to_mucontext(context)->uar.pfn,
PAGE_SIZE, vma->vm_page_prot))
return -EAGAIN;
mlx4_ib_set_vma_data(vma, &mucontext->hw_bar_info[HW_BAR_DB]);
} else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) {
/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_BF].vma)
return -EINVAL;
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
if (io_remap_pfn_range(vma, vma->vm_start,
to_mucontext(context)->uar.pfn +
dev->dev->caps.num_uars,
PAGE_SIZE, vma->vm_page_prot))
return -EAGAIN;
mlx4_ib_set_vma_data(vma, &mucontext->hw_bar_info[HW_BAR_BF]);
} else if (vma->vm_pgoff == 3) {
case 3: {
struct mlx4_clock_params params;
int ret;
/* We prevent double mmaping on same context */
if (mucontext->hw_bar_info[HW_BAR_CLOCK].vma)
return -EINVAL;
ret = mlx4_get_internal_clock_params(dev->dev, &params);
if (ret)
return ret;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
if (io_remap_pfn_range(vma, vma->vm_start,
(pci_resource_start(dev->dev->persist->pdev,
params.bar) +
params.offset)
>> PAGE_SHIFT,
PAGE_SIZE, vma->vm_page_prot))
return -EAGAIN;
return rdma_user_mmap_io(
context, vma,
(pci_resource_start(dev->dev->persist->pdev,
params.bar) +
params.offset) >>
PAGE_SHIFT,
PAGE_SIZE, pgprot_noncached(vma->vm_page_prot),
NULL);
}
mlx4_ib_set_vma_data(vma,
&mucontext->hw_bar_info[HW_BAR_CLOCK]);
} else {
default:
return -EINVAL;
}
return 0;
}
static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
static int mlx4_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct mlx4_ib_pd *pd;
struct mlx4_ib_pd *pd = to_mpd(ibpd);
struct ib_device *ibdev = ibpd->device;
int err;
pd = kmalloc(sizeof *pd, GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);
err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);
if (err) {
kfree(pd);
return ERR_PTR(err);
if (err)
return err;
if (udata && ib_copy_to_udata(udata, &pd->pdn, sizeof(__u32))) {
mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
return -EFAULT;
}
if (context)
if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) {
mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
kfree(pd);
return ERR_PTR(-EFAULT);
}
return &pd->ibpd;
return 0;
}
static int mlx4_ib_dealloc_pd(struct ib_pd *pd)
static void mlx4_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
{
mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
kfree(pd);
return 0;
}
static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
{
struct mlx4_ib_xrcd *xrcd;
@ -1294,7 +1200,7 @@ err1:
return ERR_PTR(err);
}
static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
ib_destroy_cq(to_mxrcd(xrcd)->cq);
ib_dealloc_pd(to_mxrcd(xrcd)->pd);
@ -1732,7 +1638,7 @@ static int mlx4_ib_add_dont_trap_rule(struct mlx4_dev *dev,
static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
struct ib_flow_attr *flow_attr,
int domain)
int domain, struct ib_udata *udata)
{
int err = 0, i = 0, j = 0;
struct mlx4_ib_flow *mflow;
@ -1747,6 +1653,10 @@ static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
(flow_attr->type != IB_FLOW_ATTR_NORMAL))
return ERR_PTR(-EOPNOTSUPP);
if (udata &&
udata->inlen && !ib_is_udata_cleared(udata, 0, udata->inlen))
return ERR_PTR(-EOPNOTSUPP);
memset(type, 0, sizeof(type));
mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
@ -2550,6 +2460,7 @@ static void *mlx4_ib_add(struct mlx4_dev *dev)
ibdev->dev = dev;
ibdev->bond_next_port = 0;
INIT_IB_DEVICE_OPS(&ibdev->ib_dev.ops, mlx4, MLX4);
strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
ibdev->ib_dev.owner = THIS_MODULE;
ibdev->ib_dev.node_type = RDMA_NODE_IB_CA;

9
sys/dev/mlx4/mlx4_ib/mlx4_ib_mr.c
View File

@ -323,7 +323,7 @@ mlx4_free_priv_pages(struct mlx4_ib_mr *mr)
}
}
int mlx4_ib_dereg_mr(struct ib_mr *ibmr)
int mlx4_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{
struct mlx4_ib_mr *mr = to_mmr(ibmr);
int ret;
@ -383,9 +383,8 @@ int mlx4_ib_dealloc_mw(struct ib_mw *ibmw)
return 0;
}
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg)
struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(pd->device);
struct mlx4_ib_mr *mr;
@ -409,7 +408,6 @@ struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
goto err_free_mr;
mr->max_pages = max_num_sg;
err = mlx4_mr_enable(dev->dev, &mr->mmr);
if (err)
goto err_free_pl;
@ -420,6 +418,7 @@ struct ib_mr *mlx4_ib_alloc_mr(struct ib_pd *pd,
return &mr->ibmr;
err_free_pl:
mr->ibmr.device = pd->device;
mlx4_free_priv_pages(mr);
err_free_mr:
(void) mlx4_mr_free(dev->dev, &mr->mmr);

42
sys/dev/mlx4/mlx4_ib/mlx4_ib_qp.c
View File

@ -42,6 +42,7 @@
#include <rdma/ib_pack.h>
#include <rdma/ib_addr.h>
#include <rdma/ib_mad.h>
#include <rdma/uverbs_ioctl.h>
#include <dev/mlx4/cmd.h>
#include <dev/mlx4/qp.h>
@ -1022,7 +1023,7 @@ static void get_cqs(struct mlx4_ib_qp *qp,
}
static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
int is_user)
struct ib_udata *udata)
{
struct mlx4_ib_cq *send_cq, *recv_cq;
unsigned long flags;
@ -1064,7 +1065,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
list_del(&qp->qps_list);
list_del(&qp->cq_send_list);
list_del(&qp->cq_recv_list);
if (!is_user) {
if (!udata) {
__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
if (send_cq != recv_cq)
@ -1087,11 +1088,16 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
mlx4_mtt_cleanup(dev->dev, &qp->mtt);
if (is_user) {
if (qp->rq.wqe_cnt)
mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
&qp->db);
ib_umem_release(qp->umem);
if (udata) {
if (qp->rq.wqe_cnt) {
struct mlx4_ib_ucontext *mcontext =
rdma_udata_to_drv_context(
udata,
struct mlx4_ib_ucontext,
ibucontext);
mlx4_ib_db_unmap_user(mcontext, &qp->db);
}
} else {
kvfree(qp->sq.wrid);
kvfree(qp->rq.wrid);
@ -1102,6 +1108,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
if (qp->rq.wqe_cnt)
mlx4_db_free(dev->dev, &qp->db);
}
ib_umem_release(qp->umem);
del_gid_entries(qp);
}
@ -1273,7 +1280,7 @@ struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
return ibqp;
}
static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
static int _mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(qp->device);
struct mlx4_ib_qp *mqp = to_mqp(qp);
@ -1292,7 +1299,7 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
mlx4_ib_free_qp_counter(dev, mqp);
pd = get_pd(mqp);
destroy_qp_common(dev, mqp, !!pd->ibpd.uobject);
destroy_qp_common(dev, mqp, udata);
if (is_sqp(dev, mqp))
kfree(to_msqp(mqp));
@ -1302,7 +1309,7 @@ static int _mlx4_ib_destroy_qp(struct ib_qp *qp)
return 0;
}
int mlx4_ib_destroy_qp(struct ib_qp *qp)
int mlx4_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
{
struct mlx4_ib_qp *mqp = to_mqp(qp);
@ -1313,7 +1320,7 @@ int mlx4_ib_destroy_qp(struct ib_qp *qp)
ib_destroy_qp(sqp->roce_v2_gsi);
}
return _mlx4_ib_destroy_qp(qp);
return _mlx4_ib_destroy_qp(qp, udata);
}
static int to_mlx4_st(struct mlx4_ib_dev *dev, enum mlx4_ib_qp_type type)
@ -1618,12 +1625,16 @@ static u8 gid_type_to_qpc(enum ib_gid_type gid_type)
static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
const struct ib_qp_attr *attr, int attr_mask,
enum ib_qp_state cur_state, enum ib_qp_state new_state)
enum ib_qp_state cur_state,
enum ib_qp_state new_state,
struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
struct mlx4_ib_qp *qp = to_mqp(ibqp);
struct mlx4_ib_pd *pd;
struct mlx4_ib_cq *send_cq, *recv_cq;
struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct mlx4_ib_ucontext, ibucontext);
struct mlx4_qp_context *context;
enum mlx4_qp_optpar optpar = 0;
int sqd_event;
@ -1699,10 +1710,9 @@ static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
context->param3 |= cpu_to_be32(1 << 30);
}
if (qp->ibqp.uobject)
if (ucontext)
context->usr_page = cpu_to_be32(
mlx4_to_hw_uar_index(dev->dev,
to_mucontext(ibqp->uobject->context)->uar.index));
mlx4_to_hw_uar_index(dev->dev, ucontext->uar.index));
else
context->usr_page = cpu_to_be32(
mlx4_to_hw_uar_index(dev->dev, dev->priv_uar.index));
@ -2242,7 +2252,7 @@ static int _mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
goto out;
}
err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state, udata);
if (mlx4_is_bonded(dev->dev) && (attr_mask & IB_QP_PORT))
attr->port_num = 1;

84
sys/dev/mlx4/mlx4_ib/mlx4_ib_srq.c
View File

@ -37,6 +37,7 @@
#include "mlx4_ib.h"
#include <rdma/mlx4-abi.h>
#include <rdma/uverbs_ioctl.h>
static void *get_wqe(struct mlx4_ib_srq *srq, int n)
{
@ -68,12 +69,14 @@ static void mlx4_ib_srq_event(struct mlx4_srq *srq, enum mlx4_event type)
}
}
struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
int mlx4_ib_create_srq(struct ib_srq *ib_srq,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(pd->device);
struct mlx4_ib_srq *srq;
struct mlx4_ib_dev *dev = to_mdev(ib_srq->device);
struct mlx4_ib_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct mlx4_ib_ucontext, ibucontext);
struct mlx4_ib_srq *srq = to_msrq(ib_srq);
struct mlx4_wqe_srq_next_seg *next;
struct mlx4_wqe_data_seg *scatter;
u32 cqn;
@ -86,11 +89,7 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
/* Sanity check SRQ size before proceeding */
if (init_attr->attr.max_wr >= dev->dev->caps.max_srq_wqes ||
init_attr->attr.max_sge > dev->dev->caps.max_srq_sge)
return ERR_PTR(-EINVAL);
srq = kmalloc(sizeof *srq, GFP_KERNEL);
if (!srq)
return ERR_PTR(-ENOMEM);
return -EINVAL;
mutex_init(&srq->mutex);
spin_lock_init(&srq->lock);
@ -105,20 +104,16 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
buf_size = srq->msrq.max * desc_size;
if (pd->uobject) {
if (udata) {
struct mlx4_ib_create_srq ucmd;
if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
err = -EFAULT;
goto err_srq;
}
if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
return -EFAULT;
srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
buf_size, 0, 0);
if (IS_ERR(srq->umem)) {
err = PTR_ERR(srq->umem);
goto err_srq;
}
srq->umem =
ib_umem_get(&ucontext->ibucontext, ucmd.buf_addr, buf_size, 0, 0);
if (IS_ERR(srq->umem))
return PTR_ERR(srq->umem);
err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem),
ilog2(srq->umem->page_size), &srq->mtt);
@ -129,14 +124,14 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
if (err)
goto err_mtt;
err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
err = mlx4_ib_db_map_user(ucontext,
ucmd.db_addr, &srq->db);
if (err)
goto err_mtt;
} else {
err = mlx4_db_alloc(dev->dev, &srq->db, 0, GFP_KERNEL);
if (err)
goto err_srq;
return err;
*srq->db.db = 0;
@ -183,19 +178,19 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
}
cqn = (init_attr->srq_type == IB_SRQT_XRC) ?
to_mcq(init_attr->ext.xrc.cq)->mcq.cqn : 0;
to_mcq(init_attr->ext.cq)->mcq.cqn : 0;
xrcdn = (init_attr->srq_type == IB_SRQT_XRC) ?
to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn :
(u16) dev->dev->caps.reserved_xrcds;
err = mlx4_srq_alloc(dev->dev, to_mpd(pd)->pdn, cqn, xrcdn, &srq->mtt,
srq->db.dma, &srq->msrq);
err = mlx4_srq_alloc(dev->dev, to_mpd(ib_srq->pd)->pdn, cqn, xrcdn,
&srq->mtt, srq->db.dma, &srq->msrq);
if (err)
goto err_wrid;
srq->msrq.event = mlx4_ib_srq_event;
srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
if (pd->uobject)
if (udata)
if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof (__u32))) {
err = -EFAULT;
goto err_wrid;
@ -203,11 +198,11 @@ struct ib_srq *mlx4_ib_create_srq(struct ib_pd *pd,
init_attr->attr.max_wr = srq->msrq.max - 1;
return &srq->ibsrq;
return 0;
err_wrid:
if (pd->uobject)
mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
if (udata)
mlx4_ib_db_unmap_user(ucontext, &srq->db);
else
kvfree(srq->wrid);
@ -215,19 +210,15 @@ err_mtt:
mlx4_mtt_cleanup(dev->dev, &srq->mtt);
err_buf:
if (pd->uobject)
ib_umem_release(srq->umem);
else
if (!srq->umem)
mlx4_buf_free(dev->dev, buf_size, &srq->buf);
ib_umem_release(srq->umem);
err_db:
if (!pd->uobject)
if (!udata)
mlx4_db_free(dev->dev, &srq->db);
err_srq:
kfree(srq);
return ERR_PTR(err);
return err;
}
int mlx4_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@ -274,7 +265,7 @@ int mlx4_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
return 0;
}
int mlx4_ib_destroy_srq(struct ib_srq *srq)
void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
{
struct mlx4_ib_dev *dev = to_mdev(srq->device);
struct mlx4_ib_srq *msrq = to_msrq(srq);
@ -282,19 +273,20 @@ int mlx4_ib_destroy_srq(struct ib_srq *srq)
mlx4_srq_free(dev->dev, &msrq->msrq);
mlx4_mtt_cleanup(dev->dev, &msrq->mtt);
if (srq->uobject) {
mlx4_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
ib_umem_release(msrq->umem);
if (udata) {
mlx4_ib_db_unmap_user(
rdma_udata_to_drv_context(
udata,
struct mlx4_ib_ucontext,
ibucontext),
&msrq->db);
} else {
kvfree(msrq->wrid);
mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift,
&msrq->buf);
mlx4_db_free(dev->dev, &msrq->db);
}
kfree(msrq);
return 0;
ib_umem_release(msrq->umem);
}
void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index)

22
sys/dev/mlx5/device.h
View File

@ -299,6 +299,7 @@ enum {
MLX5_EVENT_QUEUE_TYPE_QP = 0,
MLX5_EVENT_QUEUE_TYPE_RQ = 1,
MLX5_EVENT_QUEUE_TYPE_SQ = 2,
MLX5_EVENT_QUEUE_TYPE_DCT = 6,
};
enum {
@ -502,7 +503,9 @@ struct mlx5_eqe_comp {
};
struct mlx5_eqe_qp_srq {
__be32 reserved[6];
__be32 reserved1[5];
u8 type;
u8 reserved2[3];
__be32 qp_srq_n;
};
@ -512,6 +515,12 @@ struct mlx5_eqe_cq_err {
u8 syndrome;
};
struct mlx5_eqe_xrq_err {
__be32 reserved1[5];
__be32 type_xrqn;
__be32 reserved2;
};
struct mlx5_eqe_port_state {
u8 reserved0[8];
u8 port;
@ -595,6 +604,11 @@ struct mlx5_eqe_general_notification_event {
u32 rsvd0[6];
};
struct mlx5_eqe_dct {
__be32 reserved[6];
__be32 dctn;
};
struct mlx5_eqe_temp_warning {
__be64 sensor_warning_msb;
__be64 sensor_warning_lsb;
@ -614,7 +628,9 @@ union ev_data {
struct mlx5_eqe_port_module_event port_module_event;
struct mlx5_eqe_vport_change vport_change;
struct mlx5_eqe_general_notification_event general_notifications;
struct mlx5_eqe_dct dct;
struct mlx5_eqe_temp_warning temp_warning;
struct mlx5_eqe_xrq_err xrq_err;
} __packed;
struct mlx5_eqe {
@ -957,6 +973,7 @@ enum mlx5_cap_type {
MLX5_CAP_DMC,
MLX5_CAP_DEC,
MLX5_CAP_TLS,
MLX5_CAP_DEV_EVENT = 0x14,
/* NUM OF CAP Types */
MLX5_CAP_NUM
};
@ -1120,6 +1137,9 @@ enum mlx5_mcam_feature_groups {
#define MLX5_CAP_TLS(mdev, cap) \
MLX5_GET(tls_capabilities, (mdev)->hca_caps_cur[MLX5_CAP_TLS], cap)
#define MLX5_CAP_DEV_EVENT(mdev, cap)\
MLX5_ADDR_OF(device_event_cap, (mdev)->hca_caps_cur[MLX5_CAP_DEV_EVENT], cap)
enum {
MLX5_CMD_STAT_OK = 0x0,
MLX5_CMD_STAT_INT_ERR = 0x1,

34
sys/dev/mlx5/driver.h
View File

@ -399,7 +399,7 @@ struct mlx5_core_sig_ctx {
enum {
MLX5_MKEY_MR = 1,
MLX5_MKEY_MW,
MLX5_MKEY_MR_USER,
MLX5_MKEY_INDIRECT_DEVX,
};
struct mlx5_core_mkey {
@ -410,20 +410,14 @@ struct mlx5_core_mkey {
u32 type;
};
struct mlx5_core_mr {
u64 iova;
u64 size;
u32 key;
u32 pd;
};
enum mlx5_res_type {
MLX5_RES_QP = MLX5_EVENT_QUEUE_TYPE_QP,
MLX5_RES_RQ = MLX5_EVENT_QUEUE_TYPE_RQ,
MLX5_RES_SQ = MLX5_EVENT_QUEUE_TYPE_SQ,
MLX5_RES_SRQ = 3,
MLX5_RES_XSRQ = 4,
MLX5_RES_DCT = 5,
MLX5_RES_XRQ = 5,
MLX5_RES_DCT = MLX5_EVENT_QUEUE_TYPE_DCT,
};
struct mlx5_core_rsc_common {
@ -477,6 +471,7 @@ struct mlx5_core_srq {
struct completion free;
};
struct mlx5_ib_dev;
struct mlx5_eq_table {
void __iomem *update_ci;
void __iomem *update_arm_ci;
@ -485,9 +480,10 @@ struct mlx5_eq_table {
struct mlx5_eq async_eq;
struct mlx5_eq cmd_eq;
int num_comp_vectors;
/* protect EQs list
*/
spinlock_t lock;
spinlock_t lock; /* protect EQs list */
struct mlx5_ib_dev *dev; /* for devx event notifier */
bool (*cb)(struct mlx5_core_dev *mdev,
uint8_t event_type, void *data);
};
struct mlx5_core_health {
@ -999,21 +995,21 @@ int mlx5_core_arm_srq(struct mlx5_core_dev *dev, struct mlx5_core_srq *srq,
void mlx5_init_mr_table(struct mlx5_core_dev *dev);
void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev);
int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
struct mlx5_core_mr *mkey,
struct mlx5_core_mkey *mkey,
struct mlx5_async_ctx *async_ctx, u32 *in,
int inlen, u32 *out, int outlen,
mlx5_async_cbk_t callback,
struct mlx5_async_work *context);
int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
struct mlx5_core_mr *mr,
struct mlx5_core_mkey *mr,
u32 *in, int inlen);
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey);
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey,
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey);
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey,
u32 *out, int outlen);
int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mr,
int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mr,
u32 *mkey);
int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn);
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn);
int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn, u16 uid);
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid);
int mlx5_core_mad_ifc(struct mlx5_core_dev *dev, const void *inb, void *outb,
u16 opmod, u8 port);
void mlx5_fwp_flush(struct mlx5_fw_page *fwp);

8
sys/dev/mlx5/mlx5_core/mlx5_eq.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -244,7 +244,11 @@ static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n",
eq->eqn, eqe_type_str(eqe->type));
switch (eqe->type) {
if (dev->priv.eq_table.cb != NULL &&
dev->priv.eq_table.cb(dev, eqe->type, &eqe->data)) {
/* FALLTHROUGH */
} else switch (eqe->type) {
case MLX5_EVENT_TYPE_COMP:
mlx5_cq_completion(dev, eqe);
break;

8
sys/dev/mlx5/mlx5_core/mlx5_fw.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -233,6 +233,12 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev)
return err;
}
if (MLX5_CAP_GEN(dev, event_cap)) {
err = mlx5_core_get_caps(dev, MLX5_CAP_DEV_EVENT);
if (err)
return err;
}
err = mlx5_core_query_special_contexts(dev);
if (err)
return err;

12
sys/dev/mlx5/mlx5_core/mlx5_mr.c
View File

@ -49,7 +49,7 @@ void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev)
}
int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
struct mlx5_core_mr *mkey,
struct mlx5_core_mkey *mkey,
struct mlx5_async_ctx *async_ctx, u32 *in,
int inlen, u32 *out, int outlen,
mlx5_async_cbk_t callback,
@ -111,7 +111,7 @@ int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev,
EXPORT_SYMBOL(mlx5_core_create_mkey_cb);
int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
struct mlx5_core_mr *mkey,
struct mlx5_core_mkey *mkey,
u32 *in, int inlen)
{
return mlx5_core_create_mkey_cb(dev, mkey, NULL, in, inlen,
@ -119,12 +119,12 @@ int mlx5_core_create_mkey(struct mlx5_core_dev *dev,
}
EXPORT_SYMBOL(mlx5_core_create_mkey);
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey)
int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey)
{
struct mlx5_mr_table *table = &dev->priv.mr_table;
u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0};
u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0};
struct mlx5_core_mr *deleted_mr;
struct mlx5_core_mkey *deleted_mr;
unsigned long flags;
spin_lock_irqsave(&table->lock, flags);
@ -142,7 +142,7 @@ int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey)
}
EXPORT_SYMBOL(mlx5_core_destroy_mkey);
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey,
int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey,
u32 *out, int outlen)
{
u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {0};
@ -155,7 +155,7 @@ int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *mkey,
}
EXPORT_SYMBOL(mlx5_core_query_mkey);
int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mr *_mkey,
int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *_mkey,
u32 *mkey)
{
u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0};

8
sys/dev/mlx5/mlx5_core/mlx5_pd.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -30,13 +30,14 @@
#include <dev/mlx5/driver.h>
#include "mlx5_core.h"
int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn)
int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn, u16 uid)
{
u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {0};
u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {0};
int err;
MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
MLX5_SET(alloc_pd_in, in, uid, uid);
err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
if (err)
@ -47,12 +48,13 @@ int mlx5_core_alloc_pd(struct mlx5_core_dev *dev, u32 *pdn)
}
EXPORT_SYMBOL(mlx5_core_alloc_pd);
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn)
int mlx5_core_dealloc_pd(struct mlx5_core_dev *dev, u32 pdn, u16 uid)
{
u32 in[MLX5_ST_SZ_DW(dealloc_pd_in)] = {0};
u32 out[MLX5_ST_SZ_DW(dealloc_pd_out)] = {0};
MLX5_SET(dealloc_pd_in, in, opcode, MLX5_CMD_OP_DEALLOC_PD);
MLX5_SET(dealloc_pd_in, in, uid, uid);
MLX5_SET(dealloc_pd_in, in, pd, pdn);
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));

2
sys/dev/mlx5/mlx5_core/mlx5_tls.c
View File

@ -116,5 +116,5 @@ int mlx5_tls_open_tis(struct mlx5_core_dev *mdev, int tc, int tdn, int pdn, u32
void mlx5_tls_close_tis(struct mlx5_core_dev *mdev, u32 tisn)
{
mlx5_core_destroy_tis(mdev, tisn);
mlx5_core_destroy_tis(mdev, tisn, 0);
}

15
sys/dev/mlx5/mlx5_core/mlx5_transobj.c
View File

@ -30,7 +30,7 @@
#include "mlx5_core.h"
#include "transobj.h"
int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn, u32 uid)
{
u32 in[MLX5_ST_SZ_DW(alloc_transport_domain_in)] = {0};
u32 out[MLX5_ST_SZ_DW(alloc_transport_domain_out)] = {0};
@ -38,6 +38,7 @@ int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
MLX5_SET(alloc_transport_domain_in, in, opcode,
MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN);
MLX5_SET(alloc_transport_domain_in, in, uid, uid);
err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
if (!err)
@ -47,7 +48,7 @@ int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn)
return err;
}
void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn, u32 uid)
{
u32 in[MLX5_ST_SZ_DW(dealloc_transport_domain_in)] = {0};
u32 out[MLX5_ST_SZ_DW(dealloc_transport_domain_out)] = {0};
@ -55,6 +56,7 @@ void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn)
MLX5_SET(dealloc_transport_domain_in, in, opcode,
MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
MLX5_SET(dealloc_transport_domain_in, in, transport_domain, tdn);
MLX5_SET(dealloc_transport_domain_in, in, uid, uid);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
@ -164,12 +166,13 @@ int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
return err;
}
void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn)
void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u32 uid)
{
u32 in[MLX5_ST_SZ_DW(destroy_tir_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_tir_out)] = {0};
MLX5_SET(destroy_tir_in, in, opcode, MLX5_CMD_OP_DESTROY_TIR);
MLX5_SET(destroy_tir_in, in, uid, uid);
MLX5_SET(destroy_tir_in, in, tirn, tirn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
@ -201,12 +204,13 @@ int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn)
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u32 uid)
{
u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_tis_out)] = {0};
MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS);
MLX5_SET(destroy_tis_in, in, uid, uid);
MLX5_SET(destroy_tis_in, in, tisn, tisn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
@ -374,12 +378,13 @@ int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
}
void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn)
void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 uid)
{
u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
MLX5_SET(destroy_rqt_in, in, uid, uid);
MLX5_SET(destroy_rqt_in, in, rqtn, rqtn);
mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));

10
sys/dev/mlx5/mlx5_core/transobj.h
View File

@ -28,8 +28,8 @@
#ifndef __TRANSOBJ_H__
#define __TRANSOBJ_H__
int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn);
void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn);
int mlx5_alloc_transport_domain(struct mlx5_core_dev *dev, u32 *tdn, u32 uid);
void mlx5_dealloc_transport_domain(struct mlx5_core_dev *dev, u32 tdn, u32 uid);
int mlx5_core_create_rq(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rqn);
int mlx5_core_modify_rq(struct mlx5_core_dev *dev, u32 *in, int inlen);
@ -42,12 +42,12 @@ void mlx5_core_destroy_sq(struct mlx5_core_dev *dev, u32 sqn);
int mlx5_core_query_sq(struct mlx5_core_dev *dev, u32 sqn, u32 *out);
int mlx5_core_create_tir(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *tirn);
void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn);
void mlx5_core_destroy_tir(struct mlx5_core_dev *dev, u32 tirn, u32 uid);
int mlx5_core_create_tis(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *tisn);
int mlx5_core_modify_tis(struct mlx5_core_dev *dev, u32 tisn, u32 *in,
int inlen);
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn);
void mlx5_core_destroy_tis(struct mlx5_core_dev *dev, u32 tisn, u32 uid);
int mlx5_core_create_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen, u32 *rmpn);
int mlx5_core_modify_rmp(struct mlx5_core_dev *dev, u32 *in, int inlen);
int mlx5_core_destroy_rmp(struct mlx5_core_dev *dev, u32 rmpn);
@ -62,6 +62,6 @@ int mlx5_core_create_rqt(struct mlx5_core_dev *dev, u32 *in, int inlen,
u32 *rqtn);
int mlx5_core_modify_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 *in,
int inlen);
void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn);
void mlx5_core_destroy_rqt(struct mlx5_core_dev *dev, u32 rqtn, u32 uid);
#endif /* __TRANSOBJ_H__ */

2
sys/dev/mlx5/mlx5_en/en.h
View File

@ -1021,7 +1021,7 @@ struct mlx5e_priv {
struct sx state_lock; /* Protects Interface state */
u32 pdn;
u32 tdn;
struct mlx5_core_mr mr;
struct mlx5_core_mkey mr;
u32 tisn[MLX5E_MAX_TX_NUM_TC];
u32 rqtn;

18
sys/dev/mlx5/mlx5_en/mlx5_en_main.c
View File

@ -2636,7 +2636,7 @@ mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
static void
mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
{
mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc], 0);
}
static int
@ -2965,7 +2965,7 @@ static void
mlx5e_close_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
{
mlx5_core_destroy_tir(priv->mdev, inner_vxlan ?
priv->tirn_inner_vxlan[tt] : priv->tirn[tt]);
priv->tirn_inner_vxlan[tt] : priv->tirn[tt], 0);
}
static int
@ -3740,7 +3740,7 @@ mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
static int
mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
struct mlx5_core_mr *mkey)
struct mlx5_core_mkey *mkey)
{
struct ifnet *ifp = priv->ifp;
struct mlx5_core_dev *mdev = priv->mdev;
@ -4538,12 +4538,12 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
/* reuse mlx5core's watchdog workqueue */
priv->wq = mdev->priv.health.wq_watchdog;
err = mlx5_core_alloc_pd(mdev, &priv->pdn);
err = mlx5_core_alloc_pd(mdev, &priv->pdn, 0);
if (err) {
mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
goto err_free_wq;
}
err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
err = mlx5_alloc_transport_domain(mdev, &priv->tdn, 0);
if (err) {
mlx5_en_err(ifp,
"mlx5_alloc_transport_domain failed, %d\n", err);
@ -4709,10 +4709,10 @@ err_create_mkey:
mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
err_dealloc_transport_domain:
mlx5_dealloc_transport_domain(mdev, priv->tdn);
mlx5_dealloc_transport_domain(mdev, priv->tdn, 0);
err_dealloc_pd:
mlx5_core_dealloc_pd(mdev, priv->pdn);
mlx5_core_dealloc_pd(mdev, priv->pdn, 0);
err_free_wq:
flush_workqueue(priv->wq);
@ -4808,8 +4808,8 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
sysctl_ctx_free(&priv->sysctl_ctx);
mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
mlx5_dealloc_transport_domain(priv->mdev, priv->tdn, 0);
mlx5_core_dealloc_pd(priv->mdev, priv->pdn, 0);
mlx5e_disable_async_events(priv);
flush_workqueue(priv->wq);
mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);

2
sys/dev/mlx5/mlx5_en/mlx5_en_rl.c
View File

@ -634,7 +634,7 @@ mlx5e_rl_open_tis(struct mlx5e_priv *priv)
static void
mlx5e_rl_close_tis(struct mlx5e_priv *priv)
{
mlx5_core_destroy_tis(priv->mdev, priv->rl.tisn);
mlx5_core_destroy_tis(priv->mdev, priv->rl.tisn, 0);
}
static void

132
sys/dev/mlx5/mlx5_ib/mlx5_ib.h
View File

@ -33,6 +33,7 @@
#include <linux/printk.h>
#include <linux/netdevice.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_smi.h>
#include <dev/mlx5/cq.h>
#include <dev/mlx5/qp.h>
@ -41,6 +42,7 @@
#include <dev/mlx5/mlx5_core/transobj.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/mlx5-abi.h>
#include <rdma/uverbs_ioctl.h>
#define mlx5_ib_dbg(dev, format, arg...) \
pr_debug("%s:%s:%d:(pid %d): " format, (dev)->ib_dev.name, __func__, \
@ -66,15 +68,6 @@ enum {
MLX5_IB_MMAP_CMD_MASK = 0xff,
};
enum mlx5_ib_mmap_cmd {
MLX5_IB_MMAP_REGULAR_PAGE = 0,
MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES = 1,
MLX5_IB_MMAP_WC_PAGE = 2,
MLX5_IB_MMAP_NC_PAGE = 3,
/* 5 is chosen in order to be compatible with old versions of libmlx5 */
MLX5_IB_MMAP_CORE_CLOCK = 5,
};
enum {
MLX5_RES_SCAT_DATA32_CQE = 0x1,
MLX5_RES_SCAT_DATA64_CQE = 0x2,
@ -109,9 +102,11 @@ enum {
MLX5_IB_INVALID_BFREG = BIT(31),
};
struct mlx5_ib_vma_private_data {
struct list_head list;
struct vm_area_struct *vma;
enum mlx5_ib_mmap_type {
MLX5_IB_MMAP_TYPE_MEMIC = 1,
MLX5_IB_MMAP_TYPE_VAR = 2,
MLX5_IB_MMAP_TYPE_UAR_WC = 3,
MLX5_IB_MMAP_TYPE_UAR_NC = 4,
};
struct mlx5_bfreg_info {
@ -143,7 +138,9 @@ struct mlx5_ib_ucontext {
u8 cqe_version;
/* Transport Domain number */
u32 tdn;
struct list_head vma_private_list;
u64 lib_caps;
u16 devx_uid;
};
static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibucontext)
@ -154,6 +151,7 @@ static inline struct mlx5_ib_ucontext *to_mucontext(struct ib_ucontext *ibuconte
struct mlx5_ib_pd {
struct ib_pd ibpd;
u32 pdn;
u16 uid;
};
#define MLX5_IB_FLOW_MCAST_PRIO (MLX5_BY_PASS_NUM_PRIOS - 1)
@ -279,6 +277,7 @@ enum {
struct mlx5_ib_rwq_ind_table {
struct ib_rwq_ind_table ib_rwq_ind_tbl;
u32 rqtn;
u16 uid;
};
/*
@ -358,12 +357,18 @@ struct mlx5_bf {
spinlock_t lock32;
};
struct mlx5_ib_dct {
struct mlx5_core_dct mdct;
u32 *in;
};
struct mlx5_ib_qp {
struct ib_qp ibqp;
union {
struct mlx5_ib_qp_trans trans_qp;
struct mlx5_ib_raw_packet_qp raw_packet_qp;
struct mlx5_ib_rss_qp rss_qp;
struct mlx5_ib_dct dct;
};
struct mlx5_buf buf;
@ -433,6 +438,7 @@ enum mlx5_ib_qp_flags {
MLX5_IB_QP_SQPN_QP1 = 1 << 6,
MLX5_IB_QP_CAP_SCATTER_FCS = 1 << 7,
MLX5_IB_QP_RSS = 1 << 8,
MLX5_IB_QP_UNDERLAY = 1 << 10,
};
struct mlx5_umr_wr {
@ -517,6 +523,13 @@ enum mlx5_ib_mtt_access_flags {
MLX5_IB_MTT_WRITE = (1 << 1),
};
struct mlx5_user_mmap_entry {
struct rdma_user_mmap_entry rdma_entry;
u8 mmap_flag;
u64 address;
u32 page_idx;
};
#define MLX5_IB_MTT_PRESENT (MLX5_IB_MTT_READ | MLX5_IB_MTT_WRITE)
struct mlx5_ib_mr {
@ -527,7 +540,7 @@ struct mlx5_ib_mr {
int max_descs;
int desc_size;
int access_mode;
struct mlx5_core_mr mmkey;
struct mlx5_core_mkey mmkey;
struct ib_umem *umem;
struct mlx5_shared_mr_info *smr_info;
struct list_head list;
@ -545,7 +558,12 @@ struct mlx5_ib_mr {
struct mlx5_ib_mw {
struct ib_mw ibmw;
struct mlx5_core_mr mmkey;
struct mlx5_core_mkey mmkey;
};
struct mlx5_ib_devx_mr {
struct mlx5_core_mkey mmkey;
int ndescs;
};
struct mlx5_ib_umr_context {
@ -730,6 +748,12 @@ struct mlx5_ib_congestion {
};
};
struct mlx5_devx_event_table {
/* serialize updating the event_xa */
struct mutex event_xa_lock;
struct xarray event_xa;
};
struct mlx5_ib_dev {
struct ib_device ib_dev;
struct mlx5_core_dev *mdev;
@ -739,7 +763,8 @@ struct mlx5_ib_dev {
/* serialize update of capability mask
*/
struct mutex cap_mask_mutex;
bool ib_active;
u8 ib_active:1;
u8 wc_support:1;
struct umr_common umrc;
/* sync used page count stats
*/
@ -766,6 +791,7 @@ struct mlx5_ib_dev {
struct mlx5_sq_bfreg bfreg;
struct mlx5_sq_bfreg wc_bfreg;
struct mlx5_sq_bfreg fp_bfreg;
struct mlx5_devx_event_table devx_event_table;
struct mlx5_ib_congestion congestion;
struct mlx5_async_ctx async_ctx;
@ -786,6 +812,14 @@ static inline struct mlx5_ib_dev *to_mdev(struct ib_device *ibdev)
return container_of(ibdev, struct mlx5_ib_dev, ib_dev);
}
static inline struct mlx5_ib_dev *mlx5_udata_to_mdev(struct ib_udata *udata)
{
struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
return to_mdev(context->ibucontext.device);
}
static inline struct mlx5_ib_cq *to_mcq(struct ib_cq *ibcq)
{
return container_of(ibcq, struct mlx5_ib_cq, ibcq);
@ -801,7 +835,7 @@ static inline struct mlx5_ib_rwq *to_mibrwq(struct mlx5_core_qp *core_qp)
return container_of(core_qp, struct mlx5_ib_rwq, core_qp);
}
static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mr *mmkey)
static inline struct mlx5_ib_mr *to_mibmr(struct mlx5_core_mkey *mmkey)
{
return container_of(mmkey, struct mlx5_ib_mr, mmkey);
}
@ -856,6 +890,13 @@ static inline struct mlx5_ib_ah *to_mah(struct ib_ah *ibah)
return container_of(ibah, struct mlx5_ib_ah, ibah);
}
static inline struct mlx5_user_mmap_entry *
to_mmmap(struct rdma_user_mmap_entry *rdma_entry)
{
return container_of(rdma_entry,
struct mlx5_user_mmap_entry, rdma_entry);
}
int mlx5_ib_db_map_user(struct mlx5_ib_ucontext *context, unsigned long virt,
struct mlx5_db *db);
void mlx5_ib_db_unmap_user(struct mlx5_ib_ucontext *context, struct mlx5_db *db);
@ -865,17 +906,16 @@ void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index);
int mlx5_MAD_IFC(struct mlx5_ib_dev *dev, int ignore_mkey, int ignore_bkey,
u8 port, const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const void *in_mad, void *response_mad);
struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
int mlx5_ib_create_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr, u32 flags,
struct ib_udata *udata);
int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr);
int mlx5_ib_destroy_ah(struct ib_ah *ah);
struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata);
void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags);
int mlx5_ib_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *init_attr,
struct ib_udata *udata);
int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
int mlx5_ib_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
int mlx5_ib_destroy_srq(struct ib_srq *srq);
void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata);
int mlx5_ib_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
const struct ib_recv_wr **bad_wr);
struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
@ -885,7 +925,7 @@ int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata);
int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
struct ib_qp_init_attr *qp_init_attr);
int mlx5_ib_destroy_qp(struct ib_qp *qp);
int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata);
int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
const struct ib_send_wr **bad_wr);
int mlx5_ib_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
@ -894,11 +934,9 @@ void *mlx5_get_send_wqe(struct mlx5_ib_qp *qp, int n);
int mlx5_ib_read_user_wqe(struct mlx5_ib_qp *qp, int send, int wqe_index,
void *buffer, u32 length,
struct mlx5_ib_qp_base *base);
struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata);
int mlx5_ib_destroy_cq(struct ib_cq *cq);
int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata);
int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period);
@ -915,10 +953,9 @@ int mlx5_ib_update_mtt(struct mlx5_ib_mr *mr, u64 start_page_index,
int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
u64 length, u64 virt_addr, int access_flags,
struct ib_pd *pd, struct ib_udata *udata);
int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg);
int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata);
int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
unsigned int *sg_offset);
int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
@ -927,9 +964,8 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
struct ib_mad_hdr *out, size_t *out_mad_size,
u16 *out_mad_pkey_index);
struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata);
int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd);
struct ib_udata *udata);
int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata);
int mlx5_ib_get_buf_offset(u64 addr, int page_shift, u32 *offset);
int mlx5_query_ext_port_caps(struct mlx5_ib_dev *dev, u8 port);
int mlx5_query_mad_ifc_smp_attr_node_info(struct ib_device *ibdev,
@ -971,7 +1007,7 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
struct ib_wq_init_attr *init_attr,
struct ib_udata *udata);
int mlx5_ib_destroy_wq(struct ib_wq *wq);
void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata);
int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
u32 wq_attr_mask, struct ib_udata *udata);
struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
@ -1043,6 +1079,28 @@ int mlx5_ib_generate_wc(struct ib_cq *ibcq, struct ib_wc *wc);
void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi,
int bfregn);
#if 1 /* IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) */
int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user);
void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid);
void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev);
void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev);
bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type);
bool mlx5_ib_devx_is_flow_counter(void *obj, u32 offset, u32 *counter_id);
#else
static inline int
mlx5_ib_devx_create(struct mlx5_ib_dev *dev,
bool is_user) { return -EOPNOTSUPP; }
static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {}
static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {}
static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {}
static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id,
int *dest_type)
{
return false;
}
#endif
static inline void init_query_mad(struct ib_smp *mad)
{
mad->base_version = 1;

38
sys/dev/mlx5/mlx5_ib/mlx5_ib_ah.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -27,7 +27,7 @@
#include "mlx5_ib.h"
static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
static void create_ib_ah(struct mlx5_ib_dev *dev,
struct mlx5_ib_ah *ah,
struct ib_ah_attr *ah_attr,
enum rdma_link_layer ll)
@ -55,22 +55,20 @@ static struct ib_ah *create_ib_ah(struct mlx5_ib_dev *dev,
ah->av.fl_mlid = ah_attr->src_path_bits & 0x7f;
ah->av.stat_rate_sl |= (ah_attr->sl & 0xf);
}
return &ah->ibah;
}
struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
struct ib_udata *udata)
int mlx5_ib_create_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr,
u32 flags, struct ib_udata *udata)
{
struct mlx5_ib_ah *ah;
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_ah *ah = to_mah(ibah);
struct mlx5_ib_dev *dev = to_mdev(ibah->device);
enum rdma_link_layer ll;
ll = pd->device->get_link_layer(pd->device, ah_attr->port_num);
ll = dev->ib_dev.get_link_layer(&dev->ib_dev, ah_attr->port_num);
if (ll == IB_LINK_LAYER_ETHERNET && !(ah_attr->ah_flags & IB_AH_GRH))
return ERR_PTR(-EINVAL);
return -EINVAL;
if (ll == IB_LINK_LAYER_ETHERNET && udata) {
int err;
@ -79,25 +77,22 @@ struct ib_ah *mlx5_ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
sizeof(resp.dmac);
if (udata->outlen < min_resp_len)
return ERR_PTR(-EINVAL);
return -EINVAL;
resp.response_length = min_resp_len;
err = ib_resolve_eth_dmac(pd->device, ah_attr);
err = ib_resolve_eth_dmac(&dev->ib_dev, ah_attr);
if (err)
return ERR_PTR(err);
return err;
memcpy(resp.dmac, ah_attr->dmac, ETH_ALEN);
err = ib_copy_to_udata(udata, &resp, resp.response_length);
if (err)
return ERR_PTR(err);
return err;
}
ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
if (!ah)
return ERR_PTR(-ENOMEM);
return create_ib_ah(dev, ah, ah_attr, ll); /* never fails */
create_ib_ah(dev, ah, ah_attr, ll);
return 0;
}
int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
@ -123,8 +118,7 @@ int mlx5_ib_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
return 0;
}
int mlx5_ib_destroy_ah(struct ib_ah *ah)
void mlx5_ib_destroy_ah(struct ib_ah *ah, u32 flags)
{
kfree(to_mah(ah));
return 0;
return;
}

88
sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -29,6 +29,7 @@
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_cache.h>
#include <rdma/uverbs_ioctl.h>
#include "mlx5_ib.h"
static void mlx5_ib_cq_comp(struct mlx5_core_cq *cq, struct mlx5_eqe *eqe __unused)
@ -519,7 +520,7 @@ static int mlx5_poll_one(struct mlx5_ib_cq *cq,
struct mlx5_core_qp *mqp;
struct mlx5_ib_wq *wq;
struct mlx5_sig_err_cqe *sig_err_cqe;
struct mlx5_core_mr *mmkey;
struct mlx5_core_mkey *mmkey;
struct mlx5_ib_mr *mr;
unsigned long flags;
uint8_t opcode;
@ -741,11 +742,10 @@ static int alloc_cq_buf(struct mlx5_ib_dev *dev, struct mlx5_ib_cq_buf *buf,
}
static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
struct ib_ucontext *context, struct mlx5_ib_cq *cq,
int entries, u32 **cqb,
struct mlx5_ib_cq *cq, int entries, u32 **cqb,
int *cqe_size, int *index, int *inlen)
{
struct mlx5_ib_create_cq ucmd;
struct mlx5_ib_create_cq ucmd = {};
size_t ucmdlen;
int page_shift;
__be64 *pas;
@ -753,6 +753,8 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
int ncont;
void *cqc;
int err;
struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
ucmdlen = min(udata->inlen, sizeof(ucmd));
if (ucmdlen < offsetof(struct mlx5_ib_create_cq, flags))
@ -769,7 +771,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
*cqe_size = ucmd.cqe_size;
cq->buf.umem = ib_umem_get(context, ucmd.buf_addr,
cq->buf.umem = ib_umem_get(&context->ibucontext, ucmd.buf_addr,
entries * ucmd.cqe_size,
IB_ACCESS_LOCAL_WRITE, 1);
if (IS_ERR(cq->buf.umem)) {
@ -777,7 +779,7 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
return err;
}
err = mlx5_ib_db_map_user(to_mucontext(context), ucmd.db_addr,
err = mlx5_ib_db_map_user(context, ucmd.db_addr,
&cq->db);
if (err)
goto err_umem;
@ -804,29 +806,33 @@ static int create_cq_user(struct mlx5_ib_dev *dev, struct ib_udata *udata,
if (ucmd.flags & MLX5_IB_CREATE_CQ_FLAGS_UAR_PAGE_INDEX) {
*index = ucmd.uar_page_index;
} else if (to_mucontext(context)->bfregi.lib_uar_dyn) {
} else if (context->bfregi.lib_uar_dyn) {
err = -EINVAL;
goto err_cqb;
} else {
*index = to_mucontext(context)->bfregi.sys_pages[0];
*index = context->bfregi.sys_pages[0];
}
MLX5_SET(create_cq_in, *cqb, uid, context->devx_uid);
return 0;
err_cqb:
kvfree(*cqb);
err_db:
mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
mlx5_ib_db_unmap_user(context, &cq->db);
err_umem:
ib_umem_release(cq->buf.umem);
return err;
}
static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_ucontext *context)
static void destroy_cq_user(struct mlx5_ib_cq *cq, struct ib_udata *udata)
{
mlx5_ib_db_unmap_user(to_mucontext(context), &cq->db);
struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
mlx5_ib_db_unmap_user(context, &cq->db);
ib_umem_release(cq->buf.umem);
}
@ -906,16 +912,15 @@ static void notify_soft_wc_handler(struct work_struct *work)
cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
}
struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)
int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
int vector = attr->comp_vector;
struct mlx5_ib_dev *dev = to_mdev(ibdev);
u32 out[MLX5_ST_SZ_DW(create_cq_out)];
struct mlx5_ib_cq *cq;
struct mlx5_ib_cq *cq = to_mcq(ibcq);
int uninitialized_var(index);
int uninitialized_var(inlen);
u32 *cqb = NULL;
@ -927,18 +932,14 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
if (entries < 0 ||
(entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))))
return ERR_PTR(-EINVAL);
return -EINVAL;
if (check_cq_create_flags(attr->flags))
return ERR_PTR(-EOPNOTSUPP);
return -EOPNOTSUPP;
entries = roundup_pow_of_two(entries + 1);
if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))
return ERR_PTR(-EINVAL);
cq = kzalloc(sizeof(*cq), GFP_KERNEL);
if (!cq)
return ERR_PTR(-ENOMEM);
return -EINVAL;
cq->ibcq.cqe = entries - 1;
mutex_init(&cq->resize_mutex);
@ -949,17 +950,17 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
INIT_LIST_HEAD(&cq->list_send_qp);
INIT_LIST_HEAD(&cq->list_recv_qp);
if (context) {
err = create_cq_user(dev, udata, context, cq, entries,
&cqb, &cqe_size, &index, &inlen);
if (udata) {
err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size,
&index, &inlen);
if (err)
goto err_create;
return err;
} else {
cqe_size = cache_line_size() == 128 ? 128 : 64;
err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb,
&index, &inlen);
if (err)
goto err_create;
return err;
INIT_WORK(&cq->notify_work, notify_soft_wc_handler);
}
@ -990,7 +991,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
INIT_LIST_HEAD(&cq->wc_list);
if (context)
if (udata)
if (ib_copy_to_udata(udata, &cq->mcq.cqn, sizeof(__u32))) {
err = -EFAULT;
goto err_cmd;
@ -998,43 +999,30 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev,
kvfree(cqb);
return &cq->ibcq;
return 0;
err_cmd:
mlx5_core_destroy_cq(dev->mdev, &cq->mcq);
err_cqb:
kvfree(cqb);
if (context)
destroy_cq_user(cq, context);
if (udata)
destroy_cq_user(cq, udata);
else
destroy_cq_kernel(dev, cq);
err_create:
kfree(cq);
return ERR_PTR(err);
return err;
}
int mlx5_ib_destroy_cq(struct ib_cq *cq)
void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(cq->device);
struct mlx5_ib_cq *mcq = to_mcq(cq);
struct ib_ucontext *context = NULL;
if (cq->uobject)
context = cq->uobject->context;
mlx5_core_destroy_cq(dev->mdev, &mcq->mcq);
if (context)
destroy_cq_user(mcq, context);
if (udata)
destroy_cq_user(mcq, udata);
else
destroy_cq_kernel(dev, mcq);
kfree(mcq);
return 0;
}
static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn)

2930
sys/dev/mlx5/mlx5_ib/mlx5_ib_devx.c Normal file
View File

File diff suppressed because it is too large Load Diff

2
sys/dev/mlx5/mlx5_ib/mlx5_ib_gsi.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2016, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2016-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions

525
sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c
View File

@ -45,6 +45,7 @@
#include <linux/list.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
#include <rdma/uverbs_ioctl.h>
#include <linux/in.h>
#include <linux/etherdevice.h>
#include <dev/mlx5/fs.h>
@ -1106,6 +1107,12 @@ out:
return err;
}
static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
{
mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
}
static u16 calc_dynamic_bfregs(int uars_per_sys_page)
{
/* Large page with non 4k uar support might limit the dynamic size */
@ -1194,55 +1201,70 @@ static void deallocate_uars(struct mlx5_ib_dev *dev,
mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
}
static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata)
static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
u16 uid)
{
int err;
if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
return 0;
err = mlx5_alloc_transport_domain(dev->mdev, tdn, uid);
if (err)
return err;
return 0;
}
static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
u16 uid)
{
if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
return;
mlx5_dealloc_transport_domain(dev->mdev, tdn, uid);
}
static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata)
{
struct ib_device *ibdev = uctx->device;
struct mlx5_ib_dev *dev = to_mdev(ibdev);
struct mlx5_ib_alloc_ucontext_req_v2 req = {};
struct mlx5_ib_alloc_ucontext_resp resp = {};
struct mlx5_ib_ucontext *context;
struct mlx5_ib_ucontext *context = to_mucontext(uctx);
struct mlx5_bfreg_info *bfregi;
int ver;
int err;
size_t reqlen;
size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
max_cqe_version);
bool lib_uar_4k;
bool lib_uar_dyn;
if (!dev->ib_active)
return ERR_PTR(-EAGAIN);
return -EAGAIN;
if (udata->inlen < sizeof(struct ib_uverbs_cmd_hdr))
return ERR_PTR(-EINVAL);
reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
ver = 0;
else if (reqlen >= min_req_v2)
else if (udata->inlen >= min_req_v2)
ver = 2;
else
return ERR_PTR(-EINVAL);
return -EINVAL;
err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
if (err)
return ERR_PTR(err);
return err;
if (req.flags)
return ERR_PTR(-EINVAL);
if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
return -EOPNOTSUPP;
if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
return ERR_PTR(-EOPNOTSUPP);
return -EOPNOTSUPP;
req.total_num_bfregs = ALIGN(req.total_num_bfregs,
MLX5_NON_FP_BFREGS_PER_UAR);
if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
return ERR_PTR(-EINVAL);
if (reqlen > sizeof(req) &&
!ib_is_udata_cleared(udata, sizeof(req),
reqlen - sizeof(req)))
return ERR_PTR(-EOPNOTSUPP);
return -EINVAL;
resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
@ -1263,10 +1285,6 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
resp.response_length = min(offsetof(typeof(resp), response_length) +
sizeof(resp.response_length), udata->outlen);
context = kzalloc(sizeof(*context), GFP_KERNEL);
if (!context)
return ERR_PTR(-ENOMEM);
lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
bfregi = &context->bfregi;
@ -1303,19 +1321,18 @@ static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
goto out_sys_pages;
uar_done:
#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
#endif
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain)) {
err = mlx5_alloc_transport_domain(dev->mdev,
&context->tdn);
if (err)
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
err = mlx5_ib_devx_create(dev, true);
if (err < 0)
goto out_uars;
context->devx_uid = err;
}
INIT_LIST_HEAD(&context->vma_private_list);
err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
context->devx_uid);
if (err)
goto out_devx;
INIT_LIST_HEAD(&context->db_page_list);
mutex_init(&context->db_page_mutex);
@ -1360,17 +1377,21 @@ uar_done:
err = ib_copy_to_udata(udata, &resp, resp.response_length);
if (err)
goto out_td;
goto out_mdev;
bfregi->ver = ver;
bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
context->cqe_version = resp.cqe_version;
context->lib_caps = req.lib_caps;
print_lib_caps(dev, context->lib_caps);
return &context->ibucontext;
return 0;
out_td:
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_dealloc_transport_domain(dev->mdev, context->tdn);
out_mdev:
mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
out_devx:
if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
mlx5_ib_devx_destroy(dev, context->devx_uid);
out_uars:
deallocate_uars(dev, context);
@ -1382,26 +1403,24 @@ out_count:
kfree(bfregi->count);
out_ctx:
kfree(context);
return ERR_PTR(err);
return err;
}
static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
{
struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
struct mlx5_bfreg_info *bfregi;
bfregi = &context->bfregi;
if (MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
mlx5_dealloc_transport_domain(dev->mdev, context->tdn);
mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
if (context->devx_uid)
mlx5_ib_devx_destroy(dev, context->devx_uid);
deallocate_uars(dev, context);
kfree(bfregi->sys_pages);
kfree(bfregi->count);
kfree(context);
return 0;
}
static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
@ -1435,131 +1454,9 @@ static int get_extended_index(unsigned long offset)
return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
}
static void mlx5_ib_vma_open(struct vm_area_struct *area)
{
/* vma_open is called when a new VMA is created on top of our VMA. This
* is done through either mremap flow or split_vma (usually due to
* mlock, madvise, munmap, etc.) We do not support a clone of the VMA,
* as this VMA is strongly hardware related. Therefore we set the
* vm_ops of the newly created/cloned VMA to NULL, to prevent it from
* calling us again and trying to do incorrect actions. We assume that
* the original VMA size is exactly a single page, and therefore all
* "splitting" operation will not happen to it.
*/
area->vm_ops = NULL;
}
static void mlx5_ib_vma_close(struct vm_area_struct *area)
{
struct mlx5_ib_vma_private_data *mlx5_ib_vma_priv_data;
/* It's guaranteed that all VMAs opened on a FD are closed before the
* file itself is closed, therefore no sync is needed with the regular
* closing flow. (e.g. mlx5 ib_dealloc_ucontext)
* However need a sync with accessing the vma as part of
* mlx5_ib_disassociate_ucontext.
* The close operation is usually called under mm->mmap_sem except when
* process is exiting.
* The exiting case is handled explicitly as part of
* mlx5_ib_disassociate_ucontext.
*/
mlx5_ib_vma_priv_data = (struct mlx5_ib_vma_private_data *)area->vm_private_data;
/* setting the vma context pointer to null in the mlx5_ib driver's
* private data, to protect a race condition in
* mlx5_ib_disassociate_ucontext().
*/
mlx5_ib_vma_priv_data->vma = NULL;
list_del(&mlx5_ib_vma_priv_data->list);
kfree(mlx5_ib_vma_priv_data);
}
static const struct vm_operations_struct mlx5_ib_vm_ops = {
.open = mlx5_ib_vma_open,
.close = mlx5_ib_vma_close
};
static int mlx5_ib_set_vma_data(struct vm_area_struct *vma,
struct mlx5_ib_ucontext *ctx)
{
struct mlx5_ib_vma_private_data *vma_prv;
struct list_head *vma_head = &ctx->vma_private_list;
vma_prv = kzalloc(sizeof(*vma_prv), GFP_KERNEL);
if (!vma_prv)
return -ENOMEM;
vma_prv->vma = vma;
vma->vm_private_data = vma_prv;
vma->vm_ops = &mlx5_ib_vm_ops;
list_add(&vma_prv->list, vma_head);
return 0;
}
static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
{
int ret;
struct vm_area_struct *vma;
struct mlx5_ib_vma_private_data *vma_private, *n;
struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
struct task_struct *owning_process = NULL;
struct mm_struct *owning_mm = NULL;
owning_process = get_pid_task(ibcontext->tgid, PIDTYPE_PID);
if (!owning_process)
return;
owning_mm = get_task_mm(owning_process);
if (!owning_mm) {
pr_info("no mm, disassociate ucontext is pending task termination\n");
while (1) {
put_task_struct(owning_process);
usleep_range(1000, 2000);
owning_process = get_pid_task(ibcontext->tgid,
PIDTYPE_PID);
if (!owning_process || owning_process->task_thread->
td_proc->p_state == PRS_ZOMBIE) {
pr_info("disassociate ucontext done, task was terminated\n");
/* in case task was dead need to release the
* task struct.
*/
if (owning_process)
put_task_struct(owning_process);
return;
}
}
}
/* need to protect from a race on closing the vma as part of
* mlx5_ib_vma_close.
*/
down_write(&owning_mm->mmap_sem);
list_for_each_entry_safe(vma_private, n, &context->vma_private_list,
list) {
vma = vma_private->vma;
ret = zap_vma_ptes(vma, vma->vm_start,
PAGE_SIZE);
if (ret == -ENOTSUP) {
if (bootverbose)
WARN_ONCE(
"%s: zap_vma_ptes not implemented for unmanaged mappings", __func__);
} else {
WARN(ret, "%s: zap_vma_ptes failed, error %d",
__func__, -ret);
}
/* context going to be destroyed, should
* not access ops any more.
*/
/* XXXKIB vma->vm_flags &= ~(VM_SHARED | VM_MAYSHARE); */
vma->vm_ops = NULL;
list_del(&vma_private->list);
kfree(vma_private);
}
up_write(&owning_mm->mmap_sem);
mmput(owning_mm);
put_task_struct(owning_process);
}
static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
@ -1576,6 +1473,39 @@ static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
}
}
static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
struct vm_area_struct *vma,
struct mlx5_ib_ucontext *context)
{
if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
!(vma->vm_flags & VM_SHARED))
return -EINVAL;
if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
return -EOPNOTSUPP;
if (vma->vm_flags & (VM_WRITE | VM_EXEC))
return -EPERM;
return -EOPNOTSUPP;
}
static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
{
struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
switch (mentry->mmap_flag) {
case MLX5_IB_MMAP_TYPE_UAR_WC:
case MLX5_IB_MMAP_TYPE_UAR_NC:
mlx5_cmd_free_uar(dev->mdev, mentry->page_idx);
kfree(mentry);
break;
default:
WARN_ON(true);
}
}
static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
struct vm_area_struct *vma,
struct mlx5_ib_ucontext *context)
@ -1587,7 +1517,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
pgprot_t prot;
u32 bfreg_dyn_idx = 0;
u32 uar_index;
int dyn_uar = (cmd == MLX5_IB_MMAP_WC_PAGE);
int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
bfregi->num_static_sys_pages;
@ -1610,6 +1540,7 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
switch (cmd) {
case MLX5_IB_MMAP_WC_PAGE:
case MLX5_IB_MMAP_ALLOC_WC:
case MLX5_IB_MMAP_REGULAR_PAGE:
/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
prot = pgprot_writecombine(vma->vm_page_prot);
@ -1657,18 +1588,18 @@ static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
pfn = uar_index2pfn(dev, uar_index);
mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
vma->vm_page_prot = prot;
err = io_remap_pfn_range(vma, vma->vm_start, pfn,
PAGE_SIZE, vma->vm_page_prot);
err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
prot, NULL);
if (err) {
mlx5_ib_err(dev, "io_remap_pfn_range failed with error=%d, vm_start=0x%llx, pfn=%pa, mmap_cmd=%s\n",
err, (unsigned long long)vma->vm_start, &pfn, mmap_cmd2str(cmd));
mlx5_ib_err(dev,
"rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
err, mmap_cmd2str(cmd));
goto err;
}
if (dyn_uar)
bfregi->sys_pages[idx] = uar_index;
return mlx5_ib_set_vma_data(vma, context);
return 0;
err:
if (!dyn_uar)
@ -1682,6 +1613,48 @@ free_bfreg:
return err;
}
static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
{
unsigned long idx;
u8 command;
command = get_command(vma->vm_pgoff);
idx = get_extended_index(vma->vm_pgoff);
return (command << 16 | idx);
}
static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
struct vm_area_struct *vma,
struct ib_ucontext *ucontext)
{
struct mlx5_user_mmap_entry *mentry;
struct rdma_user_mmap_entry *entry;
unsigned long pgoff;
pgprot_t prot;
phys_addr_t pfn;
int ret;
pgoff = mlx5_vma_to_pgoff(vma);
entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
if (!entry)
return -EINVAL;
mentry = to_mmmap(entry);
pfn = (mentry->address >> PAGE_SHIFT);
if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
prot = pgprot_noncached(vma->vm_page_prot);
else
prot = pgprot_writecombine(vma->vm_page_prot);
ret = rdma_user_mmap_io(ucontext, vma, pfn,
entry->npages * PAGE_SIZE,
prot,
entry);
rdma_user_mmap_entry_put(&mentry->rdma_entry);
return ret;
}
static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
{
struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
@ -1692,6 +1665,10 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
command = get_command(vma->vm_pgoff);
switch (command) {
case MLX5_IB_MMAP_WC_PAGE:
case MLX5_IB_MMAP_ALLOC_WC:
if (!dev->wc_support)
return -EPERM;
/* FALLTHROUGH */
case MLX5_IB_MMAP_NC_PAGE:
case MLX5_IB_MMAP_REGULAR_PAGE:
return uar_mmap(dev, command, vma, context);
@ -1710,65 +1687,55 @@ static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vm
if (PAGE_SIZE > 4096)
return -EOPNOTSUPP;
vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
pfn = (dev->mdev->iseg_base +
offsetof(struct mlx5_init_seg, internal_timer_h)) >>
PAGE_SHIFT;
if (io_remap_pfn_range(vma, vma->vm_start, pfn,
PAGE_SIZE, vma->vm_page_prot))
return -EAGAIN;
mlx5_ib_dbg(dev, "mapped internal timer at 0x%llx, PA 0x%llx\n",
(unsigned long long)vma->vm_start,
(unsigned long long)pfn << PAGE_SHIFT);
break;
return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
PAGE_SIZE,
pgprot_noncached(vma->vm_page_prot),
NULL);
case MLX5_IB_MMAP_CLOCK_INFO:
return mlx5_ib_mmap_clock_info_page(dev, vma, context);
default:
return -EINVAL;
return mlx5_ib_mmap_offset(dev, vma, ibcontext);
}
return 0;
}
static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct mlx5_ib_pd *pd = to_mpd(ibpd);
struct ib_device *ibdev = ibpd->device;
struct mlx5_ib_alloc_pd_resp resp;
struct mlx5_ib_pd *pd;
int err;
struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
u16 uid = context ? context->devx_uid : 0;
pd = kmalloc(sizeof(*pd), GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);
err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn, uid);
if (err)
return (err);
err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
if (err) {
kfree(pd);
return ERR_PTR(err);
}
if (context) {
pd->uid = uid;
if (udata) {
resp.pdn = pd->pdn;
if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
kfree(pd);
return ERR_PTR(-EFAULT);
mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
return -EFAULT;
}
}
return &pd->ibpd;
return 0;
}
static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
{
struct mlx5_ib_dev *mdev = to_mdev(pd->device);
struct mlx5_ib_pd *mpd = to_mpd(pd);
mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
kfree(mpd);
return 0;
mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
}
enum {
@ -2358,7 +2325,8 @@ err:
static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
struct ib_flow_attr *flow_attr,
int domain)
int domain,
struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_ib_qp *mqp = to_mqp(qp);
@ -2372,6 +2340,7 @@ static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
return ERR_PTR(-ENOSPC);
if (domain != IB_FLOW_DOMAIN_USER ||
udata != NULL ||
flow_attr->port > MLX5_CAP_GEN(dev->mdev, num_ports) ||
(flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP))
return ERR_PTR(-EINVAL);
@ -2734,9 +2703,12 @@ static void destroy_umrc_res(struct mlx5_ib_dev *dev)
if (err)
mlx5_ib_warn(dev, "mr cache cleanup failed\n");
mlx5_ib_destroy_qp(dev->umrc.qp);
ib_free_cq(dev->umrc.cq);
ib_dealloc_pd(dev->umrc.pd);
if (dev->umrc.qp)
mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
if (dev->umrc.cq)
ib_free_cq(dev->umrc.cq);
if (dev->umrc.pd)
ib_dealloc_pd(dev->umrc.pd);
}
enum {
@ -2835,13 +2807,16 @@ static int create_umr_res(struct mlx5_ib_dev *dev)
return 0;
error_4:
mlx5_ib_destroy_qp(qp);
mlx5_ib_destroy_qp(qp, NULL);
dev->umrc.qp = NULL;
error_3:
ib_free_cq(cq);
dev->umrc.cq = NULL;
error_2:
ib_dealloc_pd(pd);
dev->umrc.pd = NULL;
error_0:
kfree(attr);
@ -2853,36 +2828,42 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
{
struct ib_srq_init_attr attr;
struct mlx5_ib_dev *dev;
struct ib_device *ibdev;
struct ib_cq_init_attr cq_attr = {.cqe = 1};
int port;
int ret = 0;
dev = container_of(devr, struct mlx5_ib_dev, devr);
ibdev = &dev->ib_dev;
mutex_init(&devr->mutex);
devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
if (IS_ERR(devr->p0)) {
ret = PTR_ERR(devr->p0);
goto error0;
}
devr->p0->device = &dev->ib_dev;
devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
if (!devr->p0)
return -ENOMEM;
devr->p0->device = ibdev;
devr->p0->uobject = NULL;
atomic_set(&devr->p0->usecnt, 0);
devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
if (IS_ERR(devr->c0)) {
ret = PTR_ERR(devr->c0);
ret = mlx5_ib_alloc_pd(devr->p0, NULL);
if (ret)
goto error0;
devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
if (!devr->c0) {
ret = -ENOMEM;
goto error1;
}
devr->c0->device = &dev->ib_dev;
devr->c0->uobject = NULL;
devr->c0->comp_handler = NULL;
devr->c0->event_handler = NULL;
devr->c0->cq_context = NULL;
devr->c0->device = &dev->ib_dev;
atomic_set(&devr->c0->usecnt, 0);
devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
if (ret)
goto err_create_cq;
devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
if (IS_ERR(devr->x0)) {
ret = PTR_ERR(devr->x0);
goto error2;
@ -2893,7 +2874,7 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
mutex_init(&devr->x0->tgt_qp_mutex);
INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
if (IS_ERR(devr->x1)) {
ret = PTR_ERR(devr->x1);
goto error3;
@ -2908,24 +2889,26 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
attr.attr.max_sge = 1;
attr.attr.max_wr = 1;
attr.srq_type = IB_SRQT_XRC;
attr.ext.xrc.cq = devr->c0;
attr.ext.cq = devr->c0;
attr.ext.xrc.xrcd = devr->x0;
devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
if (IS_ERR(devr->s0)) {
ret = PTR_ERR(devr->s0);
devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
if (!devr->s0) {
ret = -ENOMEM;
goto error4;
}
devr->s0->device = &dev->ib_dev;
devr->s0->pd = devr->p0;
devr->s0->uobject = NULL;
devr->s0->event_handler = NULL;
devr->s0->srq_context = NULL;
devr->s0->srq_type = IB_SRQT_XRC;
devr->s0->ext.xrc.xrcd = devr->x0;
devr->s0->ext.xrc.cq = devr->c0;
devr->s0->ext.cq = devr->c0;
ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
if (ret)
goto err_create;
atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
atomic_inc(&devr->s0->ext.cq->usecnt);
atomic_inc(&devr->p0->usecnt);
atomic_set(&devr->s0->usecnt, 0);
@ -2933,20 +2916,23 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
attr.attr.max_sge = 1;
attr.attr.max_wr = 1;
attr.srq_type = IB_SRQT_BASIC;
devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
if (IS_ERR(devr->s1)) {
ret = PTR_ERR(devr->s1);
devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
if (!devr->s1) {
ret = -ENOMEM;
goto error5;
}
devr->s1->device = &dev->ib_dev;
devr->s1->pd = devr->p0;
devr->s1->uobject = NULL;
devr->s1->event_handler = NULL;
devr->s1->srq_context = NULL;
devr->s1->srq_type = IB_SRQT_BASIC;
devr->s1->ext.xrc.cq = devr->c0;
devr->s1->ext.cq = devr->c0;
ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
if (ret)
goto error6;
atomic_inc(&devr->p0->usecnt);
atomic_set(&devr->s0->usecnt, 0);
atomic_set(&devr->s1->usecnt, 0);
for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
INIT_WORK(&devr->ports[port].pkey_change_work,
@ -2956,35 +2942,44 @@ static int create_dev_resources(struct mlx5_ib_resources *devr)
return 0;
error6:
kfree(devr->s1);
error5:
mlx5_ib_destroy_srq(devr->s0);
mlx5_ib_destroy_srq(devr->s0, NULL);
err_create:
kfree(devr->s0);
error4:
mlx5_ib_dealloc_xrcd(devr->x1);
mlx5_ib_dealloc_xrcd(devr->x1, NULL);
error3:
mlx5_ib_dealloc_xrcd(devr->x0);
mlx5_ib_dealloc_xrcd(devr->x0, NULL);
error2:
mlx5_ib_destroy_cq(devr->c0);
mlx5_ib_destroy_cq(devr->c0, NULL);
err_create_cq:
kfree(devr->c0);
error1:
mlx5_ib_dealloc_pd(devr->p0);
mlx5_ib_dealloc_pd(devr->p0, NULL);
error0:
kfree(devr->p0);
return ret;
}
static void destroy_dev_resources(struct mlx5_ib_resources *devr)
{
struct mlx5_ib_dev *dev =
container_of(devr, struct mlx5_ib_dev, devr);
int port;
mlx5_ib_destroy_srq(devr->s1);
mlx5_ib_destroy_srq(devr->s0);
mlx5_ib_dealloc_xrcd(devr->x0);
mlx5_ib_dealloc_xrcd(devr->x1);
mlx5_ib_destroy_cq(devr->c0);
mlx5_ib_dealloc_pd(devr->p0);
mlx5_ib_destroy_srq(devr->s1, NULL);
kfree(devr->s1);
mlx5_ib_destroy_srq(devr->s0, NULL);
kfree(devr->s0);
mlx5_ib_dealloc_xrcd(devr->x0, NULL);
mlx5_ib_dealloc_xrcd(devr->x1, NULL);
mlx5_ib_destroy_cq(devr->c0, NULL);
kfree(devr->c0);
mlx5_ib_dealloc_pd(devr->p0, NULL);
kfree(devr->p0);
/* Make sure no change P_Key work items are still executing */
for (port = 0; port < dev->num_ports; ++port)
for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
cancel_work_sync(&devr->ports[port].pkey_change_work);
}
@ -3296,6 +3291,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
INIT_IB_DEVICE_OPS(&dev->ib_dev.ops, mlx5, MLX5);
snprintf(dev->ib_dev.name, IB_DEVICE_NAME_MAX, "mlx5_%d", device_get_unit(mdev->pdev->dev.bsddev));
dev->ib_dev.owner = THIS_MODULE;
dev->ib_dev.node_type = RDMA_NODE_IB_CA;
@ -3353,6 +3349,7 @@ static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
dev->ib_dev.mmap = mlx5_ib_mmap;
dev->ib_dev.mmap_free = mlx5_ib_mmap_free;
dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
dev->ib_dev.create_ah = mlx5_ib_create_ah;

8
sys/dev/mlx5/mlx5_ib/mlx5_ib_mr.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -1303,7 +1303,7 @@ static int clean_mr(struct mlx5_ib_mr *mr)
return 0;
}
int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(ibmr->device);
struct mlx5_ib_mr *mr = to_mmr(ibmr);
@ -1344,7 +1344,7 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg)
u32 max_num_sg, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
@ -1389,7 +1389,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
goto err_free_in;
mr->desc_size = sizeof(struct mlx5_klm);
mr->max_descs = ndescs;
} else if (mr_type == IB_MR_TYPE_SIGNATURE) {
} else if (mr_type == IB_MR_TYPE_INTEGRITY) {
u32 psv_index[2];
MLX5_SET(mkc, mkc, bsf_en, 1);

175
sys/dev/mlx5/mlx5_ib/mlx5_ib_qp.c
View File

@ -29,6 +29,7 @@
#include <rdma/ib_umem.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_ioctl.h>
#include "mlx5_ib.h"
/* not supported currently */
@ -679,11 +680,15 @@ err_umem:
return err;
}
static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq)
static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq,
struct ib_udata *udata)
{
struct mlx5_ib_ucontext *context;
struct mlx5_ib_ucontext *context =
rdma_udata_to_drv_context(
udata,
struct mlx5_ib_ucontext,
ibucontext);
context = to_mucontext(pd->uobject->context);
mlx5_ib_db_unmap_user(context, &rwq->db);
if (rwq->umem)
ib_umem_release(rwq->umem);
@ -770,6 +775,7 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
__be64 *pas;
void *qpc;
int err;
u16 uid;
u32 uar_flags;
err = ib_copy_from_udata(&ucmd, udata, sizeof(ucmd));
@ -837,6 +843,9 @@ static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
goto err_umem;
}
uid = (attr->qp_type != IB_QPT_XRC_TGT &&
attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
MLX5_SET(create_qp_in, *in, uid, uid);
pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
if (ubuffer->umem)
mlx5_ib_populate_pas(dev, ubuffer->umem, page_shift, pas, 0);
@ -885,11 +894,15 @@ err_bfreg:
}
static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd, struct mlx5_ib_qp *qp,
struct mlx5_ib_qp_base *base)
struct mlx5_ib_qp_base *base,
struct ib_udata *udata)
{
struct mlx5_ib_ucontext *context;
struct mlx5_ib_ucontext *context =
rdma_udata_to_drv_context(
udata,
struct mlx5_ib_ucontext,
ibucontext);
context = to_mucontext(pd->uobject->context);
mlx5_ib_db_unmap_user(context, &qp->db);
if (base->ubuffer.umem)
ib_umem_release(base->ubuffer.umem);
@ -1045,19 +1058,21 @@ static int is_connected(enum ib_qp_type qp_type)
}
static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
struct mlx5_ib_sq *sq, u32 tdn)
struct mlx5_ib_sq *sq, u32 tdn,
struct ib_pd *pd)
{
u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
MLX5_SET(tisc, tisc, transport_domain, tdn);
return mlx5_core_create_tis(dev->mdev, in, sizeof(in), &sq->tisn);
}
static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
struct mlx5_ib_sq *sq)
struct mlx5_ib_sq *sq, struct ib_pd *pd)
{
mlx5_core_destroy_tis(dev->mdev, sq->tisn);
mlx5_core_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
}
static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
@ -1093,6 +1108,7 @@ static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
goto err_umem;
}
MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
MLX5_SET(sqc, sqc, flush_in_error_en, 1);
MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
@ -1154,7 +1170,8 @@ static int get_rq_pas_size(void *qpc)
}
static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
struct mlx5_ib_rq *rq, void *qpin)
struct mlx5_ib_rq *rq, void *qpin,
struct ib_pd *pd)
{
struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
__be64 *pas;
@ -1175,6 +1192,7 @@ static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
if (!in)
return -ENOMEM;
MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
MLX5_SET(rqc, rqc, vlan_strip_disable, 1);
MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_RQ_TYPE_MEMORY_RQ_INLINE);
@ -1216,7 +1234,8 @@ static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
}
static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
struct mlx5_ib_rq *rq, u32 tdn)
struct mlx5_ib_rq *rq, u32 tdn,
struct ib_pd *pd)
{
u32 *in;
void *tirc;
@ -1228,6 +1247,7 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
if (!in)
return -ENOMEM;
MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
@ -1241,9 +1261,10 @@ static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
}
static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
struct mlx5_ib_rq *rq)
struct mlx5_ib_rq *rq,
struct ib_pd *pd)
{
mlx5_core_destroy_tir(dev->mdev, rq->tirn);
mlx5_core_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
}
static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
@ -1260,7 +1281,7 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
u32 tdn = mucontext->tdn;
if (qp->sq.wqe_cnt) {
err = create_raw_packet_qp_tis(dev, sq, tdn);
err = create_raw_packet_qp_tis(dev, sq, tdn, pd);
if (err)
return err;
@ -1274,12 +1295,12 @@ static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
if (qp->rq.wqe_cnt) {
rq->base.container_mibqp = qp;
err = create_raw_packet_qp_rq(dev, rq, in);
err = create_raw_packet_qp_rq(dev, rq, in, pd);
if (err)
goto err_destroy_sq;
err = create_raw_packet_qp_tir(dev, rq, tdn);
err = create_raw_packet_qp_tir(dev, rq, tdn, pd);
if (err)
goto err_destroy_rq;
}
@ -1296,7 +1317,7 @@ err_destroy_sq:
return err;
destroy_raw_packet_qp_sq(dev, sq);
err_destroy_tis:
destroy_raw_packet_qp_tis(dev, sq);
destroy_raw_packet_qp_tis(dev, sq, pd);
return err;
}
@ -1309,13 +1330,13 @@ static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
if (qp->rq.wqe_cnt) {
destroy_raw_packet_qp_tir(dev, rq);
destroy_raw_packet_qp_tir(dev, rq, qp->ibqp.pd);
destroy_raw_packet_qp_rq(dev, rq);
}
if (qp->sq.wqe_cnt) {
destroy_raw_packet_qp_sq(dev, sq);
destroy_raw_packet_qp_tis(dev, sq);
destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
}
}
@ -1333,7 +1354,8 @@ static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
{
mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn);
mlx5_core_destroy_tir(dev->mdev, qp->rss_qp.tirn,
to_mpd(qp->ibqp.pd)->uid);
}
static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
@ -1405,6 +1427,7 @@ static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
if (!in)
return -ENOMEM;
MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
MLX5_SET(tirc, tirc, disp_type,
MLX5_TIRC_DISP_TYPE_INDIRECT);
@ -1854,7 +1877,7 @@ static int create_qp_common(struct mlx5_ib_dev *dev, struct ib_pd *pd,
err_create:
if (qp->create_type == MLX5_QP_USER)
destroy_qp_user(dev, pd, qp, base);
destroy_qp_user(dev, pd, qp, base, udata);
else if (qp->create_type == MLX5_QP_KERNEL)
destroy_qp_kernel(dev, qp);
@ -1964,7 +1987,8 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
const struct mlx5_modify_raw_qp_param *raw_qp_param,
u8 lag_tx_affinity);
static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
struct ib_udata *udata)
{
struct mlx5_ib_cq *send_cq, *recv_cq;
struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
@ -2033,7 +2057,7 @@ static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
if (qp->create_type == MLX5_QP_KERNEL)
destroy_qp_kernel(dev, qp);
else if (qp->create_type == MLX5_QP_USER)
destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base);
destroy_qp_user(dev, &get_pd(qp)->ibpd, qp, base, udata);
}
static const char *ib_qp_type_str(enum ib_qp_type type)
@ -2163,7 +2187,7 @@ struct ib_qp *mlx5_ib_create_qp(struct ib_pd *pd,
return &qp->ibqp;
}
int mlx5_ib_destroy_qp(struct ib_qp *qp)
int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(qp->device);
struct mlx5_ib_qp *mqp = to_mqp(qp);
@ -2171,7 +2195,7 @@ int mlx5_ib_destroy_qp(struct ib_qp *qp)
if (unlikely(qp->qp_type == IB_QPT_GSI))
return mlx5_ib_gsi_destroy_qp(qp);
destroy_qp_common(dev, mqp);
destroy_qp_common(dev, mqp, udata);
kfree(mqp);
@ -2244,7 +2268,8 @@ static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
}
static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
struct mlx5_ib_sq *sq, u8 sl)
struct mlx5_ib_sq *sq, u8 sl,
struct ib_pd *pd)
{
void *in;
void *tisc;
@ -2257,6 +2282,7 @@ static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
return -ENOMEM;
MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
@ -2269,7 +2295,8 @@ static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
}
static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
struct mlx5_ib_sq *sq, u8 tx_affinity)
struct mlx5_ib_sq *sq, u8 tx_affinity,
struct ib_pd *pd)
{
void *in;
void *tisc;
@ -2282,6 +2309,7 @@ static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
return -ENOMEM;
MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
@ -2362,7 +2390,7 @@ static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
if ((qp->ibqp.qp_type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
return modify_raw_packet_eth_prio(dev->mdev,
&qp->raw_packet_qp.sq,
ah->sl & 0xf);
ah->sl & 0xf, qp->ibqp.pd);
return 0;
}
@ -2510,9 +2538,9 @@ static int ib_mask_to_mlx5_opt(int ib_mask)
return result;
}
static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
struct mlx5_ib_rq *rq, int new_state,
const struct mlx5_modify_raw_qp_param *raw_qp_param)
static int modify_raw_packet_qp_rq(
struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
{
void *in;
void *rqc;
@ -2526,6 +2554,7 @@ static int modify_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
MLX5_SET(modify_rq_in, in, rqn, rq->base.mqp.qpn);
MLX5_SET(modify_rq_in, in, rq_state, rq->state);
MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
MLX5_SET(rqc, rqc, state, new_state);
@ -2552,7 +2581,8 @@ out:
}
static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
struct mlx5_ib_sq *sq, int new_state)
struct mlx5_ib_sq *sq, int new_state,
struct ib_pd *pd)
{
void *in;
void *sqc;
@ -2565,6 +2595,7 @@ static int modify_raw_packet_qp_sq(struct mlx5_core_dev *dev,
return -ENOMEM;
MLX5_SET(modify_sq_in, in, sqn, sq->base.mqp.qpn);
MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
MLX5_SET(modify_sq_in, in, sq_state, sq->state);
sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
@ -2588,6 +2619,8 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
int modify_rq = !!qp->rq.wqe_cnt;
int modify_sq = !!qp->sq.wqe_cnt;
int rq_state;
int sq_state;
int err;
@ -2605,10 +2638,11 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
rq_state = MLX5_RQC_STATE_RST;
sq_state = MLX5_SQC_STATE_RST;
break;
case MLX5_CMD_OP_INIT2INIT_QP:
case MLX5_CMD_OP_INIT2RTR_QP:
case MLX5_CMD_OP_RTR2RTS_QP:
case MLX5_CMD_OP_RTS2RTS_QP:
return raw_qp_param->set_mask ? -EINVAL : 0;
case MLX5_CMD_OP_INIT2INIT_QP:
case MLX5_CMD_OP_INIT2RTR_QP:
if (raw_qp_param->set_mask)
return -EINVAL;
else
@ -2618,21 +2652,23 @@ static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
return -EINVAL;
}
if (qp->rq.wqe_cnt) {
err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param);
if (modify_rq) {
err = modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
qp->ibqp.pd);
if (err)
return err;
}
if (qp->sq.wqe_cnt) {
if (modify_sq) {
if (tx_affinity) {
err = modify_raw_packet_tx_affinity(dev->mdev, sq,
tx_affinity);
tx_affinity,
qp->ibqp.pd);
if (err)
return err;
}
return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state);
return modify_raw_packet_qp_sq(dev->mdev, sq, sq_state, qp->ibqp.pd);
}
return 0;
@ -3815,10 +3851,10 @@ static u8 get_fence(u8 fence, const struct ib_send_wr *wr)
return 0;
}
static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
struct mlx5_wqe_ctrl_seg **ctrl,
const struct ib_send_wr *wr, unsigned *idx,
int *size, int nreq, bool send_signaled, bool solicited)
static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
struct mlx5_wqe_ctrl_seg **ctrl,
const struct ib_send_wr *wr, unsigned *idx,
int *size, int nreq, int send_flags)
{
if (unlikely(mlx5_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)))
return -ENOMEM;
@ -3829,8 +3865,10 @@ static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
*(uint32_t *)(*seg + 8) = 0;
(*ctrl)->imm = send_ieth(wr);
(*ctrl)->fm_ce_se = qp->sq_signal_bits |
(send_signaled ? MLX5_WQE_CTRL_CQ_UPDATE : 0) |
(solicited ? MLX5_WQE_CTRL_SOLICITED : 0);
(send_flags & IB_SEND_SIGNALED ?
MLX5_WQE_CTRL_CQ_UPDATE : 0) |
(send_flags & IB_SEND_SOLICITED ?
MLX5_WQE_CTRL_SOLICITED : 0);
*seg += sizeof(**ctrl);
*size = sizeof(**ctrl) / 16;
@ -3838,16 +3876,6 @@ static int __begin_wqe(struct mlx5_ib_qp *qp, void **seg,
return 0;
}
static int begin_wqe(struct mlx5_ib_qp *qp, void **seg,
struct mlx5_wqe_ctrl_seg **ctrl,
const struct ib_send_wr *wr, unsigned *idx,
int *size, int nreq)
{
return __begin_wqe(qp, seg, ctrl, wr, idx, size, nreq,
wr->send_flags & IB_SEND_SIGNALED,
wr->send_flags & IB_SEND_SOLICITED);
}
static void finish_wqe(struct mlx5_ib_qp *qp,
struct mlx5_wqe_ctrl_seg *ctrl,
u8 size, unsigned idx, u64 wr_id,
@ -3929,7 +3957,7 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
goto out;
}
err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq);
err = begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, nreq, wr->send_flags);
if (err) {
mlx5_ib_warn(dev, "\n");
err = -ENOMEM;
@ -4001,8 +4029,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
* SET_PSV WQEs are not signaled and solicited
* on error
*/
err = __begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq, false, true);
err = begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq, IB_SEND_SOLICITED);
if (err) {
mlx5_ib_warn(dev, "\n");
err = -ENOMEM;
@ -4022,8 +4050,8 @@ int mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
finish_wqe(qp, ctrl, size, idx, wr->wr_id,
nreq, get_fence(fence, wr),
next_fence, MLX5_OPCODE_SET_PSV);
err = __begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq, false, true);
err = begin_wqe(qp, &seg, &ctrl, wr,
&idx, &size, nreq, wr->send_flags);
if (err) {
mlx5_ib_warn(dev, "\n");
err = -ENOMEM;
@ -4610,8 +4638,7 @@ out:
}
struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(ibdev);
struct mlx5_ib_xrcd *xrcd;
@ -4633,20 +4660,17 @@ struct ib_xrcd *mlx5_ib_alloc_xrcd(struct ib_device *ibdev,
return &xrcd->ibxrcd;
}
int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
int err;
err = mlx5_core_xrcd_dealloc(dev->mdev, xrcdn);
if (err) {
if (err)
mlx5_ib_warn(dev, "failed to dealloc xrcdn 0x%x\n", xrcdn);
return err;
}
kfree(xrcd);
return 0;
}
@ -4690,6 +4714,7 @@ static int create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
if (!in)
return -ENOMEM;
MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
MLX5_SET(rqc, rqc, mem_rq_type,
MLX5_RQC_RQ_TYPE_MEMORY_RQ_INLINE);
@ -4842,22 +4867,20 @@ struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
err_copy:
mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
err_user_rq:
destroy_user_rq(pd, rwq);
destroy_user_rq(pd, rwq, udata);
err:
kfree(rwq);
return ERR_PTR(err);
}
int mlx5_ib_destroy_wq(struct ib_wq *wq)
void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(wq->device);
struct mlx5_ib_rwq *rwq = to_mrwq(wq);
mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);
destroy_user_rq(wq->pd, rwq);
destroy_user_rq(wq->pd, rwq, udata);
kfree(rwq);
return 0;
}
struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
@ -4911,6 +4934,9 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
for (i = 0; i < sz; i++)
MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
kvfree(in);
@ -4929,7 +4955,7 @@ struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device,
return &rwq_ind_tbl->ib_rwq_ind_tbl;
err_copy:
mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
err:
kfree(rwq_ind_tbl);
return ERR_PTR(err);
@ -4940,7 +4966,7 @@ int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn);
mlx5_core_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
kfree(rwq_ind_tbl);
return 0;
@ -4992,6 +5018,7 @@ int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
if (wq_state == IB_WQS_ERR)
wq_state = MLX5_RQC_STATE_ERR;
MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
MLX5_SET(rqc, rqc, state, wq_state);
err = mlx5_core_modify_rq(dev->mdev, in, inlen);

132
sys/dev/mlx5/mlx5_ib/mlx5_ib_srq.c
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -31,12 +31,10 @@
#include <linux/slab.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_ioctl.h>
#include "mlx5_ib.h"
/* not supported currently */
static int srq_signature;
static void *get_wqe(struct mlx5_ib_srq *srq, int n)
{
return mlx5_buf_offset(&srq->buf, n << srq->msrq.wqe_shift);
@ -73,6 +71,8 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_create_srq ucmd = {};
struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct mlx5_ib_ucontext, ibucontext);
size_t ucmdlen;
int err;
int npages;
@ -96,17 +96,15 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
udata->inlen - sizeof(ucmd)))
return -EINVAL;
if (in->type == IB_SRQT_XRC) {
err = get_srq_user_index(to_mucontext(pd->uobject->context),
&ucmd, udata->inlen, &uidx);
if (in->type != IB_SRQT_BASIC) {
err = get_srq_user_index(ucontext, &ucmd, udata->inlen, &uidx);
if (err)
return err;
}
srq->wq_sig = !!(ucmd.flags & MLX5_SRQ_FLAG_SIGNATURE);
srq->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr, buf_size,
0, 0);
srq->umem = ib_umem_get(&ucontext->ibucontext, ucmd.buf_addr, buf_size, 0, 0);
if (IS_ERR(srq->umem)) {
mlx5_ib_dbg(dev, "failed umem get, size %d\n", buf_size);
err = PTR_ERR(srq->umem);
@ -130,8 +128,7 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
mlx5_ib_populate_pas(dev, srq->umem, page_shift, in->pas, 0);
err = mlx5_ib_db_map_user(to_mucontext(pd->uobject->context),
ucmd.db_addr, &srq->db);
err = mlx5_ib_db_map_user(ucontext, ucmd.db_addr, &srq->db);
if (err) {
mlx5_ib_dbg(dev, "map doorbell failed\n");
goto err_in;
@ -139,9 +136,11 @@ static int create_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
in->log_page_size = page_shift - MLX5_ADAPTER_PAGE_SHIFT;
in->page_offset = offset;
in->uid = (in->type != IB_SRQT_XRC) ? to_mpd(pd)->uid : 0;
if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
in->type == IB_SRQT_XRC)
in->type != IB_SRQT_BASIC)
in->user_index = uidx;
return 0;
err_in:
@ -192,16 +191,14 @@ static int create_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq,
srq->wrid = kmalloc(srq->msrq.max * sizeof(u64), GFP_KERNEL);
if (!srq->wrid) {
mlx5_ib_dbg(dev, "kmalloc failed %lu\n",
(unsigned long)(srq->msrq.max * sizeof(u64)));
err = -ENOMEM;
goto err_in;
}
srq->wq_sig = !!srq_signature;
srq->wq_sig = 0;
in->log_page_size = srq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT;
if (MLX5_CAP_GEN(dev->mdev, cqe_version) == MLX5_CQE_VERSION_V1 &&
in->type == IB_SRQT_XRC)
in->type != IB_SRQT_BASIC)
in->user_index = MLX5_IB_DEFAULT_UIDX;
return 0;
@ -217,9 +214,15 @@ err_db:
return err;
}
static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq)
static void destroy_srq_user(struct ib_pd *pd, struct mlx5_ib_srq *srq,
struct ib_udata *udata)
{
mlx5_ib_db_unmap_user(to_mucontext(pd->uobject->context), &srq->db);
mlx5_ib_db_unmap_user(
rdma_udata_to_drv_context(
udata,
struct mlx5_ib_ucontext,
ibucontext),
&srq->db);
ib_umem_release(srq->umem);
}
@ -231,16 +234,16 @@ static void destroy_srq_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_srq *srq)
mlx5_db_free(dev->mdev, &srq->db);
}
struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
int mlx5_ib_create_srq(struct ib_srq *ib_srq,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
struct mlx5_ib_srq *srq;
struct mlx5_ib_dev *dev = to_mdev(ib_srq->device);
struct mlx5_ib_srq *srq = to_msrq(ib_srq);
size_t desc_size;
size_t buf_size;
int err;
struct mlx5_srq_attr in = {0};
struct mlx5_srq_attr in = {};
__u32 max_srq_wqes = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
/* Sanity check SRQ size before proceeding */
@ -248,13 +251,9 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
mlx5_ib_dbg(dev, "max_wr %d, cap %d\n",
init_attr->attr.max_wr,
max_srq_wqes);
return ERR_PTR(-EINVAL);
return -EINVAL;
}
srq = kmalloc(sizeof(*srq), GFP_KERNEL);
if (!srq)
return ERR_PTR(-ENOMEM);
mutex_init(&srq->mutex);
spin_lock_init(&srq->lock);
srq->msrq.max = roundup_pow_of_two(init_attr->attr.max_wr + 1);
@ -262,50 +261,50 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
desc_size = sizeof(struct mlx5_wqe_srq_next_seg) +
srq->msrq.max_gs * sizeof(struct mlx5_wqe_data_seg);
if (desc_size == 0 || srq->msrq.max_gs > desc_size) {
err = -EINVAL;
goto err_srq;
}
if (desc_size == 0 || srq->msrq.max_gs > desc_size)
return -EINVAL;
desc_size = roundup_pow_of_two(desc_size);
desc_size = max_t(size_t, 32, desc_size);
if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg)) {
err = -EINVAL;
goto err_srq;
}
if (desc_size < sizeof(struct mlx5_wqe_srq_next_seg))
return -EINVAL;
srq->msrq.max_avail_gather = (desc_size - sizeof(struct mlx5_wqe_srq_next_seg)) /
sizeof(struct mlx5_wqe_data_seg);
srq->msrq.wqe_shift = ilog2(desc_size);
buf_size = srq->msrq.max * desc_size;
if (buf_size < desc_size) {
err = -EINVAL;
goto err_srq;
}
if (buf_size < desc_size)
return -EINVAL;
in.type = init_attr->srq_type;
if (pd->uobject)
err = create_srq_user(pd, srq, &in, udata, buf_size);
if (udata)
err = create_srq_user(ib_srq->pd, srq, &in, udata, buf_size);
else
err = create_srq_kernel(dev, srq, &in, buf_size);
if (err || !in.pas) {
if (err) {
mlx5_ib_warn(dev, "create srq %s failed, err %d\n",
pd->uobject ? "user" : "kernel", err);
goto err_srq;
udata ? "user" : "kernel", err);
return err;
}
in.log_size = ilog2(srq->msrq.max);
in.wqe_shift = srq->msrq.wqe_shift - 4;
if (srq->wq_sig)
in.flags |= MLX5_SRQ_FLAG_WQ_SIG;
if (init_attr->srq_type == IB_SRQT_XRC) {
in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
in.cqn = to_mcq(init_attr->ext.xrc.cq)->mcq.cqn;
} else if (init_attr->srq_type == IB_SRQT_BASIC) {
in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
}
in.pd = to_mpd(pd)->pdn;
if (init_attr->srq_type == IB_SRQT_XRC)
in.xrcd = to_mxrcd(init_attr->ext.xrc.xrcd)->xrcdn;
else
in.xrcd = to_mxrcd(dev->devr.x0)->xrcdn;
if (ib_srq_has_cq(init_attr->srq_type))
in.cqn = to_mcq(init_attr->ext.cq)->mcq.cqn;
else
in.cqn = to_mcq(dev->devr.c0)->mcq.cqn;
in.pd = to_mpd(ib_srq->pd)->pdn;
in.db_record = srq->db.dma;
err = mlx5_core_create_srq(dev->mdev, &srq->msrq, &in);
kvfree(in.pas);
@ -319,7 +318,7 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
srq->msrq.event = mlx5_ib_srq_event;
srq->ibsrq.ext.xrc.srq_num = srq->msrq.srqn;
if (pd->uobject)
if (udata)
if (ib_copy_to_udata(udata, &srq->msrq.srqn, sizeof(__u32))) {
mlx5_ib_dbg(dev, "copy to user failed\n");
err = -EFAULT;
@ -328,21 +327,18 @@ struct ib_srq *mlx5_ib_create_srq(struct ib_pd *pd,
init_attr->attr.max_wr = srq->msrq.max - 1;
return &srq->ibsrq;
return 0;
err_core:
mlx5_core_destroy_srq(dev->mdev, &srq->msrq);
err_usr_kern_srq:
if (pd->uobject)
destroy_srq_user(pd, srq);
if (udata)
destroy_srq_user(ib_srq->pd, srq, udata);
else
destroy_srq_kernel(dev, srq);
err_srq:
kfree(srq);
return ERR_PTR(err);
return err;
}
int mlx5_ib_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
@ -395,7 +391,7 @@ out_box:
return ret;
}
int mlx5_ib_destroy_srq(struct ib_srq *srq)
void mlx5_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
{
struct mlx5_ib_dev *dev = to_mdev(srq->device);
struct mlx5_ib_srq *msrq = to_msrq(srq);
@ -403,14 +399,16 @@ int mlx5_ib_destroy_srq(struct ib_srq *srq)
mlx5_core_destroy_srq(dev->mdev, &msrq->msrq);
if (srq->uobject) {
mlx5_ib_db_unmap_user(to_mucontext(srq->uobject->context), &msrq->db);
mlx5_ib_db_unmap_user(
rdma_udata_to_drv_context(
udata,
struct mlx5_ib_ucontext,
ibucontext),
&msrq->db);
ib_umem_release(msrq->umem);
} else {
destroy_srq_kernel(dev, msrq);
}
kfree(srq);
return 0;
}
void mlx5_ib_free_srq_wqe(struct mlx5_ib_srq *srq, int wqe_index)

560
sys/dev/mlx5/mlx5_ifc.h
View File

@ -1,5 +1,5 @@
/*-
* Copyright (c) 2013-2019, Mellanox Technologies, Ltd. All rights reserved.
* Copyright (c) 2013-2020, Mellanox Technologies. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@ -31,6 +31,7 @@
#include <dev/mlx5/mlx5_fpga/mlx5_ifc_fpga.h>
enum {
MLX5_EVENT_TYPE_NOTIFY_ANY = 0x0,
MLX5_EVENT_TYPE_COMP = 0x0,
MLX5_EVENT_TYPE_PATH_MIG = 0x1,
MLX5_EVENT_TYPE_COMM_EST = 0x2,
@ -51,6 +52,7 @@ enum {
MLX5_EVENT_TYPE_GPIO_EVENT = 0x15,
MLX5_EVENT_TYPE_CODING_PORT_MODULE_EVENT = 0x16,
MLX5_EVENT_TYPE_TEMP_WARN_EVENT = 0x17,
MLX5_EVENT_TYPE_XRQ_ERROR = 0x18,
MLX5_EVENT_TYPE_REMOTE_CONFIG = 0x19,
MLX5_EVENT_TYPE_CODING_DCBX_CHANGE_EVENT = 0x1e,
MLX5_EVENT_TYPE_CODING_PPS_EVENT = 0x25,
@ -83,6 +85,24 @@ enum {
MLX5_SET_HCA_CAP_OP_MOD_ATOMIC = 0x3,
};
enum {
MLX5_OBJ_TYPE_GENEVE_TLV_OPT = 0x000b,
MLX5_OBJ_TYPE_MKEY = 0xff01,
MLX5_OBJ_TYPE_QP = 0xff02,
MLX5_OBJ_TYPE_PSV = 0xff03,
MLX5_OBJ_TYPE_RMP = 0xff04,
MLX5_OBJ_TYPE_XRC_SRQ = 0xff05,
MLX5_OBJ_TYPE_RQ = 0xff06,
MLX5_OBJ_TYPE_SQ = 0xff07,
MLX5_OBJ_TYPE_TIR = 0xff08,
MLX5_OBJ_TYPE_TIS = 0xff09,
MLX5_OBJ_TYPE_DCT = 0xff0a,
MLX5_OBJ_TYPE_XRQ = 0xff0b,
MLX5_OBJ_TYPE_RQT = 0xff0e,
MLX5_OBJ_TYPE_FLOW_COUNTER = 0xff0f,
MLX5_OBJ_TYPE_CQ = 0xff10,
};
enum {
MLX5_CMD_OP_QUERY_HCA_CAP = 0x100,
MLX5_CMD_OP_QUERY_ADAPTER = 0x101,
@ -140,6 +160,16 @@ enum {
MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION = 0x714,
MLX5_CMD_OP_SET_DC_CNAK_TRACE = 0x715,
MLX5_CMD_OP_QUERY_DC_CNAK_TRACE = 0x716,
MLX5_CMD_OP_CREATE_XRQ = 0x717,
MLX5_CMD_OP_DESTROY_XRQ = 0x718,
MLX5_CMD_OP_QUERY_XRQ = 0x719,
MLX5_CMD_OP_ARM_XRQ = 0x71a,
MLX5_CMD_OP_QUERY_XRQ_DC_PARAMS_ENTRY = 0x725,
MLX5_CMD_OP_SET_XRQ_DC_PARAMS_ENTRY = 0x726,
MLX5_CMD_OP_QUERY_XRQ_ERROR_PARAMS = 0x727,
MLX5_CMD_OP_RELEASE_XRQ_ERROR = 0x729,
MLX5_CMD_OP_MODIFY_XRQ = 0x72a,
MLX5_CMD_OP_QUERY_VPORT_STATE = 0x750,
MLX5_CMD_OP_MODIFY_VPORT_STATE = 0x751,
MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT = 0x752,
@ -246,8 +276,12 @@ enum {
MLX5_CMD_OP_DEALLOC_FLOW_COUNTER = 0x93a,
MLX5_CMD_OP_QUERY_FLOW_COUNTER = 0x93b,
MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c,
MLX5_CMD_OP_ALLOC_ENCAP_HEADER = 0x93d,
MLX5_CMD_OP_DEALLOC_ENCAP_HEADER = 0x93e,
MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT = 0x93d,
MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT = 0x93e,
MLX5_CMD_OP_QUERY_PACKET_REFORMAT_CONTEXT = 0x93f,
MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT = 0x940,
MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT = 0x941,
MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT = 0x942,
MLX5_CMD_OP_FPGA_CREATE_QP = 0x960,
MLX5_CMD_OP_FPGA_MODIFY_QP = 0x961,
MLX5_CMD_OP_FPGA_QUERY_QP = 0x962,
@ -257,7 +291,16 @@ enum {
MLX5_CMD_OP_MODIFY_GENERAL_OBJ = 0xa01,
MLX5_CMD_OP_QUERY_GENERAL_OBJ = 0xa02,
MLX5_CMD_OP_DESTROY_GENERAL_OBJ = 0xa03,
MLX5_CMD_OP_CREATE_UCTX = 0xa04,
MLX5_CMD_OP_DESTROY_UCTX = 0xa06,
MLX5_CMD_OP_CREATE_UMEM = 0xa08,
MLX5_CMD_OP_DESTROY_UMEM = 0xa0a,
};
/* Valid range for general commands that don't work over an object */
enum {
MLX5_CMD_OP_GENERAL_START = 0xb00,
MLX5_CMD_OP_GENERAL_END = 0xd00,
};
enum {
@ -947,6 +990,12 @@ struct mlx5_ifc_roce_cap_bits {
u8 reserved_6[0x700];
};
struct mlx5_ifc_device_event_cap_bits {
u8 user_affiliated_events[4][0x40];
u8 user_unaffiliated_events[4][0x40];
};
enum {
MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_1_BYTE = 0x1,
MLX5_ATOMIC_CAPS_ATOMIC_SIZE_QP_2_BYTES = 0x2,
@ -1043,6 +1092,11 @@ enum {
MLX5_CMD_HCA_CAP_CMDIF_CHECKSUM_ENABLED = 0x3,
};
enum {
MLX5_UCTX_CAP_RAW_TX = 1UL << 0,
MLX5_UCTX_CAP_INTERNAL_DEV_RES = 1UL << 1,
};
enum {
MLX5_SQ_TIMESTAMP_FORMAT_CAP_FREE_RUNNING = 0x0,
MLX5_SQ_TIMESTAMP_FORMAT_CAP_REAL_TIME = 0x1,
@ -1060,7 +1114,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 log_max_srq_sz[0x8];
u8 log_max_qp_sz[0x8];
u8 reserved_1[0xb];
u8 event_cap[0x1];
u8 reserved_1[0xa];
u8 log_max_qp[0x5];
u8 reserved_2[0xb];
@ -1338,7 +1393,14 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 cqe_compression_timeout[0x10];
u8 cqe_compression_max_num[0x10];
u8 reserved_69[0x220];
u8 reserved_5e0[0xc0];
u8 uctx_cap[0x20];
u8 reserved_6c0[0xc0];
u8 vhca_tunnel_commands[0x40];
u8 reserved_at_7c0[0x40];
};
enum mlx5_flow_destination_type {
@ -1436,7 +1498,9 @@ struct mlx5_ifc_wq_bits {
u8 reserved_6[0x3];
u8 log_wq_sz[0x5];
u8 reserved_7[0x15];
u8 dbr_umem_valid[0x1];
u8 wq_umem_valid[0x1];
u8 reserved_7[0x13];
u8 single_wqe_log_num_of_strides[0x3];
u8 two_byte_shift_en[0x1];
u8 reserved_8[0x4];
@ -2117,20 +2181,10 @@ struct mlx5_ifc_qpc_bits {
u8 dc_access_key[0x40];
u8 rdma_active[0x1];
u8 comm_est[0x1];
u8 suspended[0x1];
u8 reserved_31[0x5];
u8 send_msg_psn[0x18];
u8 reserved_at_680[0x3];
u8 dbr_umem_valid[0x1];
u8 reserved_32[0x8];
u8 rcv_msg_psn[0x18];
u8 rdma_va[0x40];
u8 rdma_key[0x20];
u8 reserved_33[0x20];
u8 reserved_at_684[0xbc];
};
struct mlx5_ifc_roce_addr_layout_bits {
@ -2222,7 +2276,8 @@ struct mlx5_ifc_xrc_srqc_bits {
u8 xrcd[0x18];
u8 page_offset[0x6];
u8 reserved_2[0x2];
u8 reserved_at_46[0x1];
u8 dbr_umem_valid[0x1];
u8 cqn[0x18];
u8 reserved_3[0x20];
@ -2643,6 +2698,9 @@ enum {
MLX5_ACCESS_MODE_PA = 0x0,
MLX5_ACCESS_MODE_MTT = 0x1,
MLX5_ACCESS_MODE_KLM = 0x2,
MLX5_ACCESS_MODE_KSM = 0x3,
MLX5_ACCESS_MODE_SW_ICM = 0x4,
MLX5_ACCESS_MODE_MEMIC = 0x5,
};
struct mlx5_ifc_mkc_bits {
@ -3003,7 +3061,9 @@ enum {
struct mlx5_ifc_cqc_bits {
u8 status[0x4];
u8 reserved_0[0x4];
u8 reserved_at_4[0x2];
u8 dbr_umem_valid[0x1];
u8 reserved_at_7[0x1];
u8 cqe_sz[0x3];
u8 cc[0x1];
u8 reserved_1[0x1];
@ -3117,6 +3177,54 @@ struct mlx5_ifc_config_item_bits {
u8 crc16[0x10];
};
enum {
MLX5_XRQC_STATE_GOOD = 0x0,
MLX5_XRQC_STATE_ERROR = 0x1,
};
enum {
MLX5_XRQC_TOPOLOGY_NO_SPECIAL_TOPOLOGY = 0x0,
MLX5_XRQC_TOPOLOGY_TAG_MATCHING = 0x1,
};
enum {
MLX5_XRQC_OFFLOAD_RNDV = 0x1,
};
struct mlx5_ifc_tag_matching_topology_context_bits {
u8 log_matching_list_sz[0x4];
u8 reserved_at_4[0xc];
u8 append_next_index[0x10];
u8 sw_phase_cnt[0x10];
u8 hw_phase_cnt[0x10];
u8 reserved_at_40[0x40];
};
struct mlx5_ifc_xrqc_bits {
u8 state[0x4];
u8 rlkey[0x1];
u8 reserved_at_5[0xf];
u8 topology[0x4];
u8 reserved_at_18[0x4];
u8 offload[0x4];
u8 reserved_at_20[0x8];
u8 user_index[0x18];
u8 reserved_at_40[0x8];
u8 cqn[0x18];
u8 reserved_at_60[0xa0];
struct mlx5_ifc_tag_matching_topology_context_bits tag_matching_topology_context;
u8 reserved_at_180[0x280];
struct mlx5_ifc_wq_bits wq;
};
struct mlx5_ifc_nodnic_port_config_reg_bits {
struct mlx5_ifc_nodnic_event_word_bits event;
@ -3522,7 +3630,7 @@ struct mlx5_ifc_sqd2rts_qp_out_bits {
struct mlx5_ifc_sqd2rts_qp_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -3956,6 +4064,30 @@ struct mlx5_ifc_rst2init_qp_in_bits {
u8 reserved_5[0x80];
};
struct mlx5_ifc_query_xrq_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_at_40[0x40];
struct mlx5_ifc_xrqc_bits xrq_context;
};
struct mlx5_ifc_query_xrq_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x8];
u8 xrqn[0x18];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_resume_qp_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
@ -3995,7 +4127,7 @@ struct mlx5_ifc_query_xrc_srq_out_bits {
struct mlx5_ifc_query_xrc_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5070,6 +5202,93 @@ struct mlx5_ifc_query_dc_cnak_trace_in_bits {
u8 reserved_2[0x40];
};
struct mlx5_ifc_packet_reformat_context_in_bits {
u8 reserved_at_0[0x5];
u8 reformat_type[0x3];
u8 reserved_at_8[0xe];
u8 reformat_data_size[0xa];
u8 reserved_at_20[0x10];
u8 reformat_data[2][0x8];
u8 more_reformat_data[0][0x8];
};
struct mlx5_ifc_query_packet_reformat_context_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_at_40[0xa0];
struct mlx5_ifc_packet_reformat_context_in_bits packet_reformat_context[0];
};
struct mlx5_ifc_query_packet_reformat_context_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 packet_reformat_id[0x20];
u8 reserved_at_60[0xa0];
};
struct mlx5_ifc_alloc_packet_reformat_context_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 packet_reformat_id[0x20];
u8 reserved_at_60[0x20];
};
enum mlx5_reformat_ctx_type {
MLX5_REFORMAT_TYPE_L2_TO_VXLAN = 0x0,
MLX5_REFORMAT_TYPE_L2_TO_NVGRE = 0x1,
MLX5_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x2,
MLX5_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x3,
MLX5_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x4,
};
struct mlx5_ifc_alloc_packet_reformat_context_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0xa0];
struct mlx5_ifc_packet_reformat_context_in_bits packet_reformat_context;
};
struct mlx5_ifc_dealloc_packet_reformat_context_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_at_40[0x40];
};
struct mlx5_ifc_dealloc_packet_reformat_context_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_20[0x10];
u8 op_mod[0x10];
u8 packet_reformat_id[0x20];
u8 reserved_60[0x20];
};
struct mlx5_ifc_query_cq_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
@ -5396,7 +5615,7 @@ struct mlx5_ifc_modify_tis_bitmask_bits {
struct mlx5_ifc_modify_tis_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5430,7 +5649,7 @@ enum
struct mlx5_ifc_modify_tir_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5458,7 +5677,7 @@ struct mlx5_ifc_modify_sq_out_bits {
struct mlx5_ifc_modify_sq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5528,7 +5747,7 @@ struct mlx5_ifc_modify_rqt_out_bits {
struct mlx5_ifc_modify_rqt_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5561,7 +5780,7 @@ enum {
struct mlx5_ifc_modify_rq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5597,7 +5816,7 @@ struct mlx5_ifc_rmp_bitmask_bits {
struct mlx5_ifc_modify_rmp_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5807,7 +6026,7 @@ enum {
struct mlx5_ifc_modify_cq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -5819,7 +6038,12 @@ struct mlx5_ifc_modify_cq_in_bits {
struct mlx5_ifc_cqc_bits cq_context;
u8 reserved_3[0x600];
u8 reserved_at_280[0x60];
u8 cq_umem_valid[0x1];
u8 reserved_at_2e1[0x1f];
u8 reserved_at_300[0x580];
u8 pas[0][0x40];
};
@ -6162,7 +6386,7 @@ struct mlx5_ifc_detach_from_mcg_out_bits {
struct mlx5_ifc_detach_from_mcg_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6186,7 +6410,7 @@ struct mlx5_ifc_destroy_xrc_srq_out_bits {
struct mlx5_ifc_destroy_xrc_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6208,7 +6432,7 @@ struct mlx5_ifc_destroy_tis_out_bits {
struct mlx5_ifc_destroy_tis_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6230,7 +6454,7 @@ struct mlx5_ifc_destroy_tir_out_bits {
struct mlx5_ifc_destroy_tir_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6252,7 +6476,7 @@ struct mlx5_ifc_destroy_srq_out_bits {
struct mlx5_ifc_destroy_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6324,7 +6548,7 @@ struct mlx5_ifc_destroy_rqt_out_bits {
struct mlx5_ifc_destroy_rqt_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6609,7 +6833,7 @@ struct mlx5_ifc_destroy_cq_out_bits {
struct mlx5_ifc_destroy_cq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6712,7 +6936,7 @@ struct mlx5_ifc_dealloc_xrcd_out_bits {
struct mlx5_ifc_dealloc_xrcd_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6756,7 +6980,7 @@ struct mlx5_ifc_dealloc_transport_domain_out_bits {
struct mlx5_ifc_dealloc_transport_domain_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6878,7 +7102,7 @@ struct mlx5_ifc_dealloc_pd_out_bits {
struct mlx5_ifc_dealloc_pd_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6911,6 +7135,30 @@ struct mlx5_ifc_dealloc_flow_counter_in_bits {
u8 reserved_3[0x20];
};
struct mlx5_ifc_create_xrq_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_at_40[0x8];
u8 xrqn[0x18];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_create_xrq_in_bits {
u8 opcode[0x10];
u8 uid[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x40];
struct mlx5_ifc_xrqc_bits xrq_context;
};
struct mlx5_ifc_deactivate_tracer_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
@ -6946,7 +7194,7 @@ struct mlx5_ifc_create_xrc_srq_out_bits {
struct mlx5_ifc_create_xrc_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6955,7 +7203,12 @@ struct mlx5_ifc_create_xrc_srq_in_bits {
struct mlx5_ifc_xrc_srqc_bits xrc_srq_context_entry;
u8 reserved_3[0x600];
u8 reserved_at_280[0x60];
u8 xrc_srq_umem_valid[0x1];
u8 reserved_at_2e1[0x1f];
u8 reserved_at_300[0x580];
u8 pas[0][0x40];
};
@ -6974,7 +7227,7 @@ struct mlx5_ifc_create_tis_out_bits {
struct mlx5_ifc_create_tis_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -6998,7 +7251,7 @@ struct mlx5_ifc_create_tir_out_bits {
struct mlx5_ifc_create_tir_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7022,7 +7275,7 @@ struct mlx5_ifc_create_srq_out_bits {
struct mlx5_ifc_create_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7108,7 +7361,7 @@ struct mlx5_ifc_create_rqt_out_bits {
struct mlx5_ifc_create_rqt_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7156,7 +7409,7 @@ struct mlx5_ifc_create_rmp_out_bits {
struct mlx5_ifc_create_rmp_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7196,7 +7449,10 @@ struct mlx5_ifc_create_qp_in_bits {
struct mlx5_ifc_qpc_bits qpc;
u8 reserved_5[0x80];
u8 reserved_at_800[0x60];
u8 wq_umem_valid[0x1];
u8 reserved_at_861[0x1f];
u8 pas[0][0x40];
};
@ -7282,7 +7538,8 @@ struct mlx5_ifc_create_mkey_in_bits {
u8 reserved_2[0x20];
u8 pg_access[0x1];
u8 reserved_3[0x1f];
u8 mkey_umem_valid[0x1];
u8 reserved_at_62[0x1e];
struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
@ -7478,7 +7735,7 @@ struct mlx5_ifc_create_cq_out_bits {
struct mlx5_ifc_create_cq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7487,7 +7744,10 @@ struct mlx5_ifc_create_cq_in_bits {
struct mlx5_ifc_cqc_bits cq_context;
u8 reserved_3[0x600];
u8 reserved_at_280[0x60];
u8 cq_umem_valid[0x1];
u8 reserved_at_2e1[0x59f];
u8 pas[0][0x40];
};
@ -7535,7 +7795,7 @@ struct mlx5_ifc_attach_to_mcg_out_bits {
struct mlx5_ifc_attach_to_mcg_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7548,6 +7808,29 @@ struct mlx5_ifc_attach_to_mcg_in_bits {
u8 multicast_gid[16][0x8];
};
struct mlx5_ifc_arm_xrq_out_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_at_40[0x40];
};
struct mlx5_ifc_arm_xrq_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x8];
u8 xrqn[0x18];
u8 reserved_at_60[0x10];
u8 lwm[0x10];
};
struct mlx5_ifc_arm_xrc_srq_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
@ -7563,7 +7846,7 @@ enum {
struct mlx5_ifc_arm_xrc_srq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7590,7 +7873,7 @@ enum {
struct mlx5_ifc_arm_rq_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7638,7 +7921,7 @@ struct mlx5_ifc_alloc_xrcd_out_bits {
struct mlx5_ifc_alloc_xrcd_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7682,7 +7965,7 @@ struct mlx5_ifc_alloc_transport_domain_out_bits {
struct mlx5_ifc_alloc_transport_domain_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7704,7 +7987,7 @@ struct mlx5_ifc_alloc_q_counter_out_bits {
struct mlx5_ifc_alloc_q_counter_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7726,7 +8009,7 @@ struct mlx5_ifc_alloc_pd_out_bits {
struct mlx5_ifc_alloc_pd_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 uid[0x10];
u8 reserved_1[0x10];
u8 op_mod[0x10];
@ -7736,24 +8019,24 @@ struct mlx5_ifc_alloc_pd_in_bits {
struct mlx5_ifc_alloc_flow_counter_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 reserved_1[0x10];
u8 flow_counter_id[0x10];
u8 flow_counter_id[0x20];
u8 reserved_2[0x20];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_alloc_flow_counter_in_bits {
u8 opcode[0x10];
u8 reserved_0[0x10];
u8 reserved_at_10[0x10];
u8 reserved_1[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_2[0x40];
u8 reserved_at_40[0x38];
u8 flow_counter_bulk[0x8];
};
struct mlx5_ifc_add_vxlan_udp_dport_out_bits {
@ -7810,7 +8093,7 @@ struct mlx5_ifc_set_rate_limit_out_bits {
struct mlx5_ifc_set_rate_limit_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 uid[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
@ -10847,4 +11130,147 @@ struct mlx5_ifc_mtmp_ext_bits {
u8 sensor_name_lo[0x20];
};
struct mlx5_ifc_general_obj_in_cmd_hdr_bits {
u8 opcode[0x10];
u8 uid[0x10];
u8 vhca_tunnel_id[0x10];
u8 obj_type[0x10];
u8 obj_id[0x20];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_general_obj_out_cmd_hdr_bits {
u8 status[0x8];
u8 reserved_at_8[0x18];
u8 syndrome[0x20];
u8 obj_id[0x20];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_umem_bits {
u8 reserved_at_0[0x80];
u8 reserved_at_80[0x1b];
u8 log_page_size[0x5];
u8 page_offset[0x20];
u8 num_of_mtt[0x40];
struct mlx5_ifc_mtt_bits mtt[0];
};
struct mlx5_ifc_uctx_bits {
u8 cap[0x20];
u8 reserved_at_20[0x160];
};
struct mlx5_ifc_create_umem_in_bits {
u8 opcode[0x10];
u8 uid[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x40];
struct mlx5_ifc_umem_bits umem;
};
struct mlx5_ifc_create_uctx_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x40];
struct mlx5_ifc_uctx_bits uctx;
};
struct mlx5_ifc_destroy_uctx_in_bits {
u8 opcode[0x10];
u8 reserved_at_10[0x10];
u8 reserved_at_20[0x10];
u8 op_mod[0x10];
u8 reserved_at_40[0x10];
u8 uid[0x10];
u8 reserved_at_60[0x20];
};
struct mlx5_ifc_mtrc_string_db_param_bits {
u8 string_db_base_address[0x20];
u8 reserved_at_20[0x8];
u8 string_db_size[0x18];
};
struct mlx5_ifc_mtrc_cap_bits {
u8 trace_owner[0x1];
u8 trace_to_memory[0x1];
u8 reserved_at_2[0x4];
u8 trc_ver[0x2];
u8 reserved_at_8[0x14];
u8 num_string_db[0x4];
u8 first_string_trace[0x8];
u8 num_string_trace[0x8];
u8 reserved_at_30[0x28];
u8 log_max_trace_buffer_size[0x8];
u8 reserved_at_60[0x20];
struct mlx5_ifc_mtrc_string_db_param_bits string_db_param[8];
u8 reserved_at_280[0x180];
};
struct mlx5_ifc_mtrc_conf_bits {
u8 reserved_at_0[0x1c];
u8 trace_mode[0x4];
u8 reserved_at_20[0x18];
u8 log_trace_buffer_size[0x8];
u8 trace_mkey[0x20];
u8 reserved_at_60[0x3a0];
};
struct mlx5_ifc_mtrc_stdb_bits {
u8 string_db_index[0x4];
u8 reserved_at_4[0x4];
u8 read_size[0x18];
u8 start_offset[0x20];
u8 string_db_data[0];
};
struct mlx5_ifc_mtrc_ctrl_bits {
u8 trace_status[0x2];
u8 reserved_at_2[0x2];
u8 arm_event[0x1];
u8 reserved_at_5[0xb];
u8 modify_field_select[0x10];
u8 reserved_at_20[0x2b];
u8 current_timestamp52_32[0x15];
u8 current_timestamp31_0[0x20];
u8 reserved_at_80[0x180];
};
struct mlx5_ifc_affiliated_event_header_bits {
u8 reserved_at_0[0x10];
u8 obj_type[0x10];
u8 obj_id[0x20];
};
#endif /* MLX5_IFC_H */

2
sys/dev/mlx5/qp.h
View File

@ -563,7 +563,7 @@ static inline struct mlx5_core_qp *__mlx5_qp_lookup(struct mlx5_core_dev *dev, u
return radix_tree_lookup(&dev->priv.qp_table.tree, qpn);
}
static inline struct mlx5_core_mr *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
static inline struct mlx5_core_mkey *__mlx5_mr_lookup(struct mlx5_core_dev *dev, u32 key)
{
return radix_tree_lookup(&dev->priv.mr_table.tree, key);
}

1
sys/dev/mlx5/srq.h
View File

@ -51,6 +51,7 @@ struct mlx5_srq_attr {
u32 user_index;
u64 db_record;
u64 *pas;
u16 uid;
};
struct mlx5_core_dev;

9
sys/dev/mthca/mthca_dev.h
View File

@ -512,7 +512,8 @@ int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent
void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe);
int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
struct ib_srq_attr *attr, struct mthca_srq *srq);
struct ib_srq_attr *attr, struct mthca_srq *srq,
struct ib_udata *udata);
void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq);
int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
@ -549,7 +550,8 @@ int mthca_alloc_qp(struct mthca_dev *dev,
enum ib_qp_type type,
enum ib_sig_type send_policy,
struct ib_qp_cap *cap,
struct mthca_qp *qp);
struct mthca_qp *qp,
struct ib_udata *udata);
int mthca_alloc_sqp(struct mthca_dev *dev,
struct mthca_pd *pd,
struct mthca_cq *send_cq,
@ -558,7 +560,8 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
struct ib_qp_cap *cap,
int qpn,
int port,
struct mthca_sqp *sqp);
struct mthca_sqp *sqp,
struct ib_udata *udata);
void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp);
int mthca_create_ah(struct mthca_dev *dev,
struct mthca_pd *pd,

7
sys/dev/mthca/mthca_mad.c
View File

@ -87,13 +87,13 @@ static void update_sm_ah(struct mthca_dev *dev,
ah_attr.port_num = port_num;
new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd,
&ah_attr);
&ah_attr, 0);
if (IS_ERR(new_ah))
return;
spin_lock_irqsave(&dev->sm_lock, flags);
if (dev->sm_ah[port_num - 1])
ib_destroy_ah(dev->sm_ah[port_num - 1]);
ib_destroy_ah(dev->sm_ah[port_num - 1], 0);
dev->sm_ah[port_num - 1] = new_ah;
spin_unlock_irqrestore(&dev->sm_lock, flags);
}
@ -344,6 +344,7 @@ void mthca_free_agents(struct mthca_dev *dev)
}
if (dev->sm_ah[p])
ib_destroy_ah(dev->sm_ah[p]);
ib_destroy_ah(dev->sm_ah[p],
RDMA_DESTROY_AH_SLEEPABLE);
}
}

317
sys/dev/mthca/mthca_provider.c
View File

@ -37,6 +37,7 @@
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_ioctl.h>
#include <linux/sched.h>
#include <linux/slab.h>
@ -298,17 +299,16 @@ static int mthca_query_gid(struct ib_device *ibdev, u8 port,
return err;
}
static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata)
static int mthca_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata)
{
struct mthca_alloc_ucontext_resp uresp;
struct mthca_ucontext *context;
struct ib_device *ibdev = uctx->device;
struct mthca_alloc_ucontext_resp uresp = {};
struct mthca_ucontext *context = to_mucontext(uctx);
int err;
if (!(to_mdev(ibdev)->active))
return ERR_PTR(-EAGAIN);
memset(&uresp, 0, sizeof uresp);
return -EAGAIN;
uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps;
if (mthca_is_memfree(to_mdev(ibdev)))
@ -316,44 +316,33 @@ static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev,
else
uresp.uarc_size = 0;
context = kmalloc(sizeof *context, GFP_KERNEL);
if (!context)
return ERR_PTR(-ENOMEM);
err = mthca_uar_alloc(to_mdev(ibdev), &context->uar);
if (err) {
kfree(context);
return ERR_PTR(err);
}
if (err)
return err;
context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev));
if (IS_ERR(context->db_tab)) {
err = PTR_ERR(context->db_tab);
mthca_uar_free(to_mdev(ibdev), &context->uar);
kfree(context);
return ERR_PTR(err);
return err;
}
if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) {
if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab);
mthca_uar_free(to_mdev(ibdev), &context->uar);
kfree(context);
return ERR_PTR(-EFAULT);
return -EFAULT;
}
context->reg_mr_warned = 0;
return &context->ibucontext;
return 0;
}
static int mthca_dealloc_ucontext(struct ib_ucontext *context)
static void mthca_dealloc_ucontext(struct ib_ucontext *context)
{
mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar,
to_mucontext(context)->db_tab);
mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar);
kfree(to_mucontext(context));
return 0;
}
static int mthca_mmap_uar(struct ib_ucontext *context,
@ -372,150 +361,115 @@ static int mthca_mmap_uar(struct ib_ucontext *context,
return 0;
}
static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev,
struct ib_ucontext *context,
struct ib_udata *udata)
static int mthca_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct mthca_pd *pd;
struct ib_device *ibdev = ibpd->device;
struct mthca_pd *pd = to_mpd(ibpd);
int err;
pd = kmalloc(sizeof *pd, GFP_KERNEL);
if (!pd)
return ERR_PTR(-ENOMEM);
err = mthca_pd_alloc(to_mdev(ibdev), !udata, pd);
if (err)
return err;
err = mthca_pd_alloc(to_mdev(ibdev), !context, pd);
if (err) {
kfree(pd);
return ERR_PTR(err);
}
if (context) {
if (udata) {
if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) {
mthca_pd_free(to_mdev(ibdev), pd);
kfree(pd);
return ERR_PTR(-EFAULT);
return -EFAULT;
}
}
return &pd->ibpd;
return 0;
}
static int mthca_dealloc_pd(struct ib_pd *pd)
static void mthca_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
{
mthca_pd_free(to_mdev(pd->device), to_mpd(pd));
kfree(pd);
return 0;
}
static struct ib_ah *mthca_ah_create(struct ib_pd *pd,
struct ib_ah_attr *ah_attr,
struct ib_udata *udata)
static int mthca_ah_create(struct ib_ah *ibah,
struct ib_ah_attr *init_attr, u32 flags,
struct ib_udata *udata)
{
int err;
struct mthca_ah *ah;
struct mthca_ah *ah = to_mah(ibah);
ah = kmalloc(sizeof *ah, GFP_ATOMIC);
if (!ah)
return ERR_PTR(-ENOMEM);
err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah);
if (err) {
kfree(ah);
return ERR_PTR(err);
}
return &ah->ibah;
return mthca_create_ah(to_mdev(ibah->device), to_mpd(ibah->pd),
init_attr, ah);
}
static int mthca_ah_destroy(struct ib_ah *ah)
static void mthca_ah_destroy(struct ib_ah *ah, u32 flags)
{
mthca_destroy_ah(to_mdev(ah->device), to_mah(ah));
kfree(ah);
return 0;
}
static struct ib_srq *mthca_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
static int mthca_create_srq(struct ib_srq *ibsrq,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
{
struct mthca_create_srq ucmd;
struct mthca_ucontext *context = NULL;
struct mthca_srq *srq;
struct mthca_ucontext *context = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
struct mthca_srq *srq = to_msrq(ibsrq);
int err;
if (init_attr->srq_type != IB_SRQT_BASIC)
return ERR_PTR(-ENOSYS);
return -EOPNOTSUPP;
srq = kmalloc(sizeof *srq, GFP_KERNEL);
if (!srq)
return ERR_PTR(-ENOMEM);
if (udata) {
if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
return -EFAULT;
if (pd->uobject) {
context = to_mucontext(pd->uobject->context);
if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
err = -EFAULT;
goto err_free;
}
err = mthca_map_user_db(to_mdev(pd->device), &context->uar,
err = mthca_map_user_db(to_mdev(ibsrq->device), &context->uar,
context->db_tab, ucmd.db_index,
ucmd.db_page);
if (err)
goto err_free;
return err;
srq->mr.ibmr.lkey = ucmd.lkey;
srq->db_index = ucmd.db_index;
}
err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd),
&init_attr->attr, srq);
err = mthca_alloc_srq(to_mdev(ibsrq->device), to_mpd(ibsrq->pd),
&init_attr->attr, srq, udata);
if (err && pd->uobject)
mthca_unmap_user_db(to_mdev(pd->device), &context->uar,
if (err && udata)
mthca_unmap_user_db(to_mdev(ibsrq->device), &context->uar,
context->db_tab, ucmd.db_index);
if (err)
goto err_free;
return err;
if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof (__u32))) {
mthca_free_srq(to_mdev(pd->device), srq);
err = -EFAULT;
goto err_free;
if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof(__u32))) {
mthca_free_srq(to_mdev(ibsrq->device), srq);
return -EFAULT;
}
return &srq->ibsrq;
err_free:
kfree(srq);
return ERR_PTR(err);
return 0;
}
static int mthca_destroy_srq(struct ib_srq *srq)
static void mthca_destroy_srq(struct ib_srq *srq, struct ib_udata *udata)
{
struct mthca_ucontext *context;
if (srq->uobject) {
context = to_mucontext(srq->uobject->context);
if (udata) {
struct mthca_ucontext *context =
rdma_udata_to_drv_context(
udata,
struct mthca_ucontext,
ibucontext);
mthca_unmap_user_db(to_mdev(srq->device), &context->uar,
context->db_tab, to_msrq(srq)->db_index);
}
mthca_free_srq(to_mdev(srq->device), to_msrq(srq));
kfree(srq);
return 0;
}
static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata)
{
struct mthca_ucontext *context = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
struct mthca_create_qp ucmd;
struct mthca_qp *qp;
int err;
@ -528,15 +482,11 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
case IB_QPT_UC:
case IB_QPT_UD:
{
struct mthca_ucontext *context;
qp = kmalloc(sizeof *qp, GFP_KERNEL);
qp = kzalloc(sizeof(*qp), GFP_KERNEL);
if (!qp)
return ERR_PTR(-ENOMEM);
if (pd->uobject) {
context = to_mucontext(pd->uobject->context);
if (udata) {
if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
kfree(qp);
return ERR_PTR(-EFAULT);
@ -571,11 +521,9 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
to_mcq(init_attr->send_cq),
to_mcq(init_attr->recv_cq),
init_attr->qp_type, init_attr->sq_sig_type,
&init_attr->cap, qp);
if (err && pd->uobject) {
context = to_mucontext(pd->uobject->context);
&init_attr->cap, qp, udata);
if (err && udata) {
mthca_unmap_user_db(to_mdev(pd->device),
&context->uar,
context->db_tab,
@ -592,11 +540,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
case IB_QPT_SMI:
case IB_QPT_GSI:
{
/* Don't allow userspace to create special QPs */
if (pd->uobject)
return ERR_PTR(-EINVAL);
qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL);
qp = kzalloc(sizeof(*qp), GFP_KERNEL);
if (!qp)
return ERR_PTR(-ENOMEM);
@ -607,7 +551,7 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
to_mcq(init_attr->recv_cq),
init_attr->sq_sig_type, &init_attr->cap,
qp->ibqp.qp_num, init_attr->port_num,
to_msqp(qp));
to_msqp(qp), udata);
break;
}
default:
@ -629,64 +573,68 @@ static struct ib_qp *mthca_create_qp(struct ib_pd *pd,
return &qp->ibqp;
}
static int mthca_destroy_qp(struct ib_qp *qp)
static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
{
if (qp->uobject) {
if (udata) {
struct mthca_ucontext *context =
rdma_udata_to_drv_context(
udata,
struct mthca_ucontext,
ibucontext);
mthca_unmap_user_db(to_mdev(qp->device),
&to_mucontext(qp->uobject->context)->uar,
to_mucontext(qp->uobject->context)->db_tab,
&context->uar,
context->db_tab,
to_mqp(qp)->sq.db_index);
mthca_unmap_user_db(to_mdev(qp->device),
&to_mucontext(qp->uobject->context)->uar,
to_mucontext(qp->uobject->context)->db_tab,
&context->uar,
context->db_tab,
to_mqp(qp)->rq.db_index);
}
mthca_free_qp(to_mdev(qp->device), to_mqp(qp));
kfree(qp);
kfree(to_mqp(qp));
return 0;
}
static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *context,
struct ib_udata *udata)
static int mthca_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
struct ib_device *ibdev = ibcq->device;
int entries = attr->cqe;
struct mthca_create_cq ucmd;
struct mthca_cq *cq;
int nent;
int err;
struct mthca_ucontext *context = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
if (attr->flags)
return ERR_PTR(-EINVAL);
return -EOPNOTSUPP;
if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes)
return ERR_PTR(-EINVAL);
return -EINVAL;
if (context) {
if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd))
return ERR_PTR(-EFAULT);
if (udata) {
if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd)))
return -EFAULT;
err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
to_mucontext(context)->db_tab,
ucmd.set_db_index, ucmd.set_db_page);
err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.set_db_index,
ucmd.set_db_page);
if (err)
return ERR_PTR(err);
return err;
err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
to_mucontext(context)->db_tab,
ucmd.arm_db_index, ucmd.arm_db_page);
err = mthca_map_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.arm_db_index,
ucmd.arm_db_page);
if (err)
goto err_unmap_set;
}
cq = kmalloc(sizeof *cq, GFP_KERNEL);
if (!cq) {
err = -ENOMEM;
goto err_unmap_arm;
}
cq = to_mcq(ibcq);
if (context) {
if (udata) {
cq->buf.mr.ibmr.lkey = ucmd.lkey;
cq->set_ci_db_index = ucmd.set_db_index;
cq->arm_db_index = ucmd.arm_db_index;
@ -695,37 +643,33 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev,
for (nent = 1; nent <= entries; nent <<= 1)
; /* nothing */
err = mthca_init_cq(to_mdev(ibdev), nent,
context ? to_mucontext(context) : NULL,
context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
err = mthca_init_cq(to_mdev(ibdev), nent, context,
udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num,
cq);
if (err)
goto err_free;
goto err_unmap_arm;
if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) {
if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) {
mthca_free_cq(to_mdev(ibdev), cq);
err = -EFAULT;
goto err_free;
goto err_unmap_arm;
}
cq->resize_buf = NULL;
return &cq->ibcq;
err_free:
kfree(cq);
return 0;
err_unmap_arm:
if (context)
mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
to_mucontext(context)->db_tab, ucmd.arm_db_index);
if (udata)
mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.arm_db_index);
err_unmap_set:
if (context)
mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar,
to_mucontext(context)->db_tab, ucmd.set_db_index);
if (udata)
mthca_unmap_user_db(to_mdev(ibdev), &context->uar,
context->db_tab, ucmd.set_db_index);
return ERR_PTR(err);
return err;
}
static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq,
@ -849,22 +793,25 @@ out:
return ret;
}
static int mthca_destroy_cq(struct ib_cq *cq)
static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata)
{
if (cq->uobject) {
if (udata) {
struct mthca_ucontext *context =
rdma_udata_to_drv_context(
udata,
struct mthca_ucontext,
ibucontext);
mthca_unmap_user_db(to_mdev(cq->device),
&to_mucontext(cq->uobject->context)->uar,
to_mucontext(cq->uobject->context)->db_tab,
&context->uar,
context->db_tab,
to_mcq(cq)->arm_db_index);
mthca_unmap_user_db(to_mdev(cq->device),
&to_mucontext(cq->uobject->context)->uar,
to_mucontext(cq->uobject->context)->db_tab,
&context->uar,
context->db_tab,
to_mcq(cq)->set_ci_db_index);
}
mthca_free_cq(to_mdev(cq->device), to_mcq(cq));
kfree(cq);
return 0;
}
static inline u32 convert_access(int acc)
@ -999,13 +946,12 @@ err:
return ERR_PTR(err);
}
static int mthca_dereg_mr(struct ib_mr *mr)
static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata)
{
struct mthca_mr *mmr = to_mmr(mr);
mthca_free_mr(to_mdev(mr->device), mmr);
if (mmr->umem)
ib_umem_release(mmr->umem);
ib_umem_release(mmr->umem);
kfree(mmr);
return 0;
@ -1196,6 +1142,13 @@ int mthca_register_device(struct mthca_dev *dev)
if (ret)
return ret;
#define mthca_ib_ah mthca_ah
#define mthca_ib_cq mthca_cq
#define mthca_ib_pd mthca_pd
#define mthca_ib_qp mthca_qp
#define mthca_ib_srq mthca_srq
#define mthca_ib_ucontext mthca_ucontext
INIT_IB_DEVICE_OPS(&dev->ib_dev.ops, mthca, MTHCA);
strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX);
dev->ib_dev.owner = THIS_MODULE;

35
sys/dev/mthca/mthca_qp.c
View File

@ -42,6 +42,7 @@
#include <rdma/ib_verbs.h>
#include <rdma/ib_cache.h>
#include <rdma/ib_pack.h>
#include <rdma/uverbs_ioctl.h>
#include "mthca_dev.h"
#include "mthca_cmd.h"
@ -542,10 +543,14 @@ static int mthca_path_set(struct mthca_dev *dev, const struct ib_ah_attr *ah,
static int __mthca_modify_qp(struct ib_qp *ibqp,
const struct ib_qp_attr *attr, int attr_mask,
enum ib_qp_state cur_state, enum ib_qp_state new_state)
enum ib_qp_state cur_state,
enum ib_qp_state new_state,
struct ib_udata *udata)
{
struct mthca_dev *dev = to_mdev(ibqp->device);
struct mthca_qp *qp = to_mqp(ibqp);
struct mthca_ucontext *context = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
struct mthca_mailbox *mailbox;
struct mthca_qp_param *qp_param;
struct mthca_qp_context *qp_context;
@ -607,8 +612,7 @@ static int __mthca_modify_qp(struct ib_qp *ibqp,
/* leave arbel_sched_queue as 0 */
if (qp->ibqp.uobject)
qp_context->usr_page =
cpu_to_be32(to_mucontext(qp->ibqp.uobject->context)->uar.index);
qp_context->usr_page = cpu_to_be32(context->uar.index);
else
qp_context->usr_page = cpu_to_be32(dev->driver_uar.index);
qp_context->local_qpn = cpu_to_be32(qp->qpn);
@ -900,7 +904,8 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
goto out;
}
err = __mthca_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
err = __mthca_modify_qp(ibqp, attr, attr_mask, cur_state, new_state,
udata);
out:
mutex_unlock(&qp->mutex);
@ -968,7 +973,8 @@ static void mthca_adjust_qp_caps(struct mthca_dev *dev,
*/
static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
struct mthca_pd *pd,
struct mthca_qp *qp)
struct mthca_qp *qp,
struct ib_udata *udata)
{
int size;
int err = -ENOMEM;
@ -1035,7 +1041,7 @@ static int mthca_alloc_wqe_buf(struct mthca_dev *dev,
* allocate anything. All we need is to calculate the WQE
* sizes and the send_wqe_offset, so we're done now.
*/
if (pd->ibpd.uobject)
if (udata)
return 0;
size = PAGE_ALIGN(qp->send_wqe_offset +
@ -1141,7 +1147,8 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
struct mthca_cq *send_cq,
struct mthca_cq *recv_cq,
enum ib_sig_type send_policy,
struct mthca_qp *qp)
struct mthca_qp *qp,
struct ib_udata *udata)
{
int ret;
int i;
@ -1164,7 +1171,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
if (ret)
return ret;
ret = mthca_alloc_wqe_buf(dev, pd, qp);
ret = mthca_alloc_wqe_buf(dev, pd, qp, udata);
if (ret) {
mthca_unmap_memfree(dev, qp);
return ret;
@ -1177,7 +1184,7 @@ static int mthca_alloc_qp_common(struct mthca_dev *dev,
* will be allocated and buffers will be initialized in
* userspace.
*/
if (pd->ibpd.uobject)
if (udata)
return 0;
ret = mthca_alloc_memfree(dev, qp);
@ -1270,7 +1277,8 @@ int mthca_alloc_qp(struct mthca_dev *dev,
enum ib_qp_type type,
enum ib_sig_type send_policy,
struct ib_qp_cap *cap,
struct mthca_qp *qp)
struct mthca_qp *qp,
struct ib_udata *udata)
{
int err;
@ -1293,7 +1301,7 @@ int mthca_alloc_qp(struct mthca_dev *dev,
qp->port = 0;
err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
send_policy, qp);
send_policy, qp, udata);
if (err) {
mthca_free(&dev->qp_table.alloc, qp->qpn);
return err;
@ -1345,7 +1353,8 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
struct ib_qp_cap *cap,
int qpn,
int port,
struct mthca_sqp *sqp)
struct mthca_sqp *sqp,
struct ib_udata *udata)
{
u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1;
int err;
@ -1376,7 +1385,7 @@ int mthca_alloc_sqp(struct mthca_dev *dev,
sqp->qp.transport = MLX;
err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq,
send_policy, &sqp->qp);
send_policy, &sqp->qp, udata);
if (err)
goto err_out_free;

42
sys/dev/mthca/mthca_srq.c
View File

@ -36,6 +36,8 @@
#include <asm/io.h>
#include <rdma/uverbs_ioctl.h>
#include "mthca_dev.h"
#include "mthca_cmd.h"
#include "mthca_memfree.h"
@ -95,17 +97,20 @@ static inline int *wqe_to_link(void *wqe)
static void mthca_tavor_init_srq_context(struct mthca_dev *dev,
struct mthca_pd *pd,
struct mthca_srq *srq,
struct mthca_tavor_srq_context *context)
struct mthca_tavor_srq_context *context,
struct ib_udata *udata)
{
struct mthca_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
memset(context, 0, sizeof *context);
context->wqe_base_ds = cpu_to_be64(1 << (srq->wqe_shift - 4));
context->state_pd = cpu_to_be32(pd->pd_num);
context->lkey = cpu_to_be32(srq->mr.ibmr.lkey);
if (pd->ibpd.uobject)
context->uar =
cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index);
if (udata)
context->uar = cpu_to_be32(ucontext->uar.index);
else
context->uar = cpu_to_be32(dev->driver_uar.index);
}
@ -113,8 +118,11 @@ static void mthca_tavor_init_srq_context(struct mthca_dev *dev,
static void mthca_arbel_init_srq_context(struct mthca_dev *dev,
struct mthca_pd *pd,
struct mthca_srq *srq,
struct mthca_arbel_srq_context *context)
struct mthca_arbel_srq_context *context,
struct ib_udata *udata)
{
struct mthca_ucontext *ucontext = rdma_udata_to_drv_context(
udata, struct mthca_ucontext, ibucontext);
int logsize;
memset(context, 0, sizeof *context);
@ -123,9 +131,8 @@ static void mthca_arbel_init_srq_context(struct mthca_dev *dev,
context->lkey = cpu_to_be32(srq->mr.ibmr.lkey);
context->db_index = cpu_to_be32(srq->db_index);
context->logstride_usrpage = cpu_to_be32((srq->wqe_shift - 4) << 29);
if (pd->ibpd.uobject)
context->logstride_usrpage |=
cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index);
if (udata)
context->logstride_usrpage |= cpu_to_be32(ucontext->uar.index);
else
context->logstride_usrpage |= cpu_to_be32(dev->driver_uar.index);
context->eq_pd = cpu_to_be32(MTHCA_EQ_ASYNC << 24 | pd->pd_num);
@ -139,14 +146,14 @@ static void mthca_free_srq_buf(struct mthca_dev *dev, struct mthca_srq *srq)
}
static int mthca_alloc_srq_buf(struct mthca_dev *dev, struct mthca_pd *pd,
struct mthca_srq *srq)
struct mthca_srq *srq, struct ib_udata *udata)
{
struct mthca_data_seg *scatter;
void *wqe;
int err;
int i;
if (pd->ibpd.uobject)
if (udata)
return 0;
srq->wrid = kmalloc(srq->max * sizeof (u64), GFP_KERNEL);
@ -191,7 +198,8 @@ static int mthca_alloc_srq_buf(struct mthca_dev *dev, struct mthca_pd *pd,
}
int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
struct ib_srq_attr *attr, struct mthca_srq *srq)
struct ib_srq_attr *attr, struct mthca_srq *srq,
struct ib_udata *udata)
{
struct mthca_mailbox *mailbox;
int ds;
@ -229,7 +237,7 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
if (err)
goto err_out;
if (!pd->ibpd.uobject) {
if (!udata) {
srq->db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SRQ,
srq->srqn, &srq->db);
if (srq->db_index < 0) {
@ -245,7 +253,7 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
goto err_out_db;
}
err = mthca_alloc_srq_buf(dev, pd, srq);
err = mthca_alloc_srq_buf(dev, pd, srq, udata);
if (err)
goto err_out_mailbox;
@ -255,9 +263,9 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
mutex_init(&srq->mutex);
if (mthca_is_memfree(dev))
mthca_arbel_init_srq_context(dev, pd, srq, mailbox->buf);
mthca_arbel_init_srq_context(dev, pd, srq, mailbox->buf, udata);
else
mthca_tavor_init_srq_context(dev, pd, srq, mailbox->buf);
mthca_tavor_init_srq_context(dev, pd, srq, mailbox->buf, udata);
err = mthca_SW2HW_SRQ(dev, mailbox, srq->srqn);
@ -291,14 +299,14 @@ err_out_free_srq:
mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err);
err_out_free_buf:
if (!pd->ibpd.uobject)
if (!udata)
mthca_free_srq_buf(dev, srq);
err_out_mailbox:
mthca_free_mailbox(dev, mailbox);
err_out_db:
if (!pd->ibpd.uobject && mthca_is_memfree(dev))
if (!udata && mthca_is_memfree(dev))
mthca_free_db(dev, MTHCA_DB_TYPE_SRQ, srq->db_index);
err_out_icm:

1
sys/dev/qlnx/qlnxr/qlnxr_def.h
View File

@ -65,6 +65,7 @@
#include <rdma/ib_umem.h>
#include <rdma/ib_mad.h>
#include <rdma/ib_sa.h>
#include <rdma/uverbs_ioctl.h>
#if __FreeBSD_version < 1100000
#undef MODULE_VERSION

7
sys/dev/qlnx/qlnxr/qlnxr_os.c
View File

@ -156,6 +156,13 @@ qlnxr_register_device(qlnxr_dev_t *dev)
ibdev = &dev->ibdev;
#define qlnxr_ib_ah qlnxr_ah
#define qlnxr_ib_cq qlnxr_cq
#define qlnxr_ib_pd qlnxr_pd
#define qlnxr_ib_qp qlnxr_qp
#define qlnxr_ib_srq qlnxr_srq
#define qlnxr_ib_ucontext qlnxr_ucontext
INIT_IB_DEVICE_OPS(&ibdev->ops, qlnxr, QLNXR);
strlcpy(ibdev->name, "qlnxr%d", IB_DEVICE_NAME_MAX);
memset(&ibdev->node_guid, 0, sizeof(ibdev->node_guid));

243
sys/dev/qlnx/qlnxr/qlnxr_verbs.c
View File

@ -71,8 +71,7 @@ __FBSDID("$FreeBSD$");
((unsigned char *)&addr)[3]
static int
qlnxr_check_srq_params(struct ib_pd *ibpd,
struct qlnxr_dev *dev,
qlnxr_check_srq_params(struct qlnxr_dev *dev,
struct ib_srq_init_attr *attrs);
static int
@ -159,9 +158,10 @@ qlnxr_query_gid(struct ib_device *ibdev, u8 port, int index,
return 0;
}
struct ib_srq *
qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
int
qlnxr_create_srq(struct ib_srq *ibsrq,
struct ib_srq_init_attr *init_attr,
struct ib_udata *udata)
{
struct qlnxr_dev *dev;
qlnx_host_t *ha;
@ -169,36 +169,28 @@ qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
struct ecore_rdma_create_srq_out_params out_params;
struct ecore_rdma_create_srq_in_params in_params;
u64 pbl_base_addr, phy_prod_pair_addr;
struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
struct ib_ucontext *ib_ctx = NULL;
struct qlnxr_srq_hwq_info *hw_srq;
struct qlnxr_ucontext *ctx = NULL;
struct qlnxr_ucontext *ctx;
struct qlnxr_create_srq_ureq ureq;
u32 page_cnt, page_size;
struct qlnxr_srq *srq;
struct qlnxr_srq *srq = get_qlnxr_srq(ibsrq);
int ret = 0;
dev = get_qlnxr_dev((ibpd->device));
dev = get_qlnxr_dev(ibsrq->device);
ha = dev->ha;
QL_DPRINT12(ha, "enter\n");
ret = qlnxr_check_srq_params(ibpd, dev, init_attr);
srq = kzalloc(sizeof(*srq), GFP_KERNEL);
if (!srq) {
QL_DPRINT11(ha, "cannot allocate memory for srq\n");
return NULL; //@@@ : TODO what to return here?
}
ret = qlnxr_check_srq_params(dev, init_attr);
srq->dev = dev;
hw_srq = &srq->hw_srq;
spin_lock_init(&srq->lock);
memset(&in_params, 0, sizeof(in_params));
if (udata && ibpd->uobject && ibpd->uobject->context) {
ib_ctx = ibpd->uobject->context;
ctx = get_qlnxr_ucontext(ib_ctx);
if (udata) {
ctx = rdma_udata_to_drv_context(
udata, struct qlnxr_ucontext, ibucontext);
memset(&ureq, 0, sizeof(ureq));
if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
@ -208,7 +200,7 @@ qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
goto err0;
}
ret = qlnxr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
ret = qlnxr_init_srq_user_params(&ctx->ibucontext, srq, &ureq, 0, 0);
if (ret)
goto err0;
@ -232,7 +224,7 @@ qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
page_size = pbl->elem_per_page << 4;
}
in_params.pd_id = pd->pd_id;
in_params.pd_id = get_qlnxr_pd(ibsrq->pd)->pd_id;
in_params.pbl_base_addr = pbl_base_addr;
in_params.prod_pair_addr = phy_prod_pair_addr;
in_params.num_pages = page_cnt;
@ -251,7 +243,7 @@ qlnxr_create_srq(struct ib_pd *ibpd, struct ib_srq_init_attr *init_attr,
}
QL_DPRINT12(ha, "created srq with srq_id = 0x%0x\n", srq->srq_id);
return &srq->ibsrq;
return (0);
err2:
memset(&in_params, 0, sizeof(in_params));
destroy_in_params.srq_id = srq->srq_id;
@ -264,12 +256,11 @@ err1:
qlnxr_free_srq_kernel_params(srq);
err0:
kfree(srq);
return ERR_PTR(-EFAULT);
return (-EFAULT);
}
int
qlnxr_destroy_srq(struct ib_srq *ibsrq)
void
qlnxr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
{
struct qlnxr_dev *dev;
struct qlnxr_srq *srq;
@ -291,8 +282,6 @@ qlnxr_destroy_srq(struct ib_srq *ibsrq)
qlnxr_free_srq_kernel_params(srq);
QL_DPRINT12(ha, "destroyed srq_id=0x%0x\n", srq->srq_id);
kfree(srq);
return 0;
}
int
@ -718,11 +707,11 @@ qlnxr_link_layer(struct ib_device *ibdev, uint8_t port_num)
return IB_LINK_LAYER_ETHERNET;
}
struct ib_pd *
qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
struct ib_udata *udata)
int
qlnxr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct qlnxr_pd *pd = NULL;
struct ib_device *ibdev = ibpd->device;
struct qlnxr_pd *pd = get_qlnxr_pd(ibpd);
u16 pd_id;
int rc;
struct qlnxr_dev *dev;
@ -731,8 +720,7 @@ qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
dev = get_qlnxr_dev(ibdev);
ha = dev->ha;
QL_DPRINT12(ha, "ibdev = %p context = %p"
" udata = %p enter\n", ibdev, context, udata);
QL_DPRINT12(ha, "ibdev = %p udata = %p enter\n", ibdev, udata);
if (dev->rdma_ctx == NULL) {
QL_DPRINT11(ha, "dev->rdma_ctx = NULL\n");
@ -740,13 +728,6 @@ qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
goto err;
}
pd = kzalloc(sizeof(*pd), GFP_KERNEL);
if (!pd) {
rc = -ENOMEM;
QL_DPRINT11(ha, "kzalloc(pd) = NULL\n");
goto err;
}
rc = ecore_rdma_alloc_pd(dev->rdma_ctx, &pd_id);
if (rc) {
QL_DPRINT11(ha, "ecore_rdma_alloc_pd failed\n");
@ -755,7 +736,7 @@ qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
pd->pd_id = pd_id;
if (udata && context) {
if (udata) {
rc = ib_copy_to_udata(udata, &pd->pd_id, sizeof(pd->pd_id));
if (rc) {
QL_DPRINT11(ha, "ib_copy_to_udata failed\n");
@ -763,7 +744,8 @@ qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
goto err;
}
pd->uctx = get_qlnxr_ucontext(context);
pd->uctx = rdma_udata_to_drv_context(
udata, struct qlnxr_ucontext, ibucontext);
pd->uctx->pd = pd;
}
@ -771,16 +753,15 @@ qlnxr_alloc_pd(struct ib_device *ibdev, struct ib_ucontext *context,
QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
pd, pd_id, dev->pd_count);
return &pd->ibpd;
return (0);
err:
kfree(pd);
QL_DPRINT12(ha, "exit -1\n");
return ERR_PTR(rc);
return (rc);
}
int
qlnxr_dealloc_pd(struct ib_pd *ibpd)
void
qlnxr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
{
struct qlnxr_pd *pd;
struct qlnxr_dev *dev;
@ -796,14 +777,12 @@ qlnxr_dealloc_pd(struct ib_pd *ibpd)
QL_DPRINT11(ha, "pd = NULL\n");
} else {
ecore_rdma_free_pd(dev->rdma_ctx, pd->pd_id);
kfree(pd);
atomic_subtract_rel_32(&dev->pd_count, 1);
QL_DPRINT12(ha, "exit [pd, pd_id, pd_count] = [%p, 0x%x, %d]\n",
pd, pd->pd_id, dev->pd_count);
}
QL_DPRINT12(ha, "exit\n");
return 0;
}
#define ROCE_WQE_ELEM_SIZE sizeof(struct rdma_sq_sge)
@ -977,24 +956,18 @@ qlnxr_search_mmap(struct qlnxr_ucontext *uctx, u64 phy_addr, unsigned long len)
return found;
}
struct
ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata)
int
qlnxr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
{
int rc;
struct qlnxr_ucontext *ctx;
struct qlnxr_ucontext *ctx = get_qlnxr_ucontext(uctx);
struct qlnxr_alloc_ucontext_resp uresp;
struct qlnxr_dev *dev = get_qlnxr_dev(ibdev);
struct qlnxr_dev *dev = get_qlnxr_dev(uctx->device);
qlnx_host_t *ha = dev->ha;
struct ecore_rdma_add_user_out_params oparams;
if (!udata) {
return ERR_PTR(-EFAULT);
}
ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
if (!ctx)
return ERR_PTR(-ENOMEM);
if (!udata)
return -EFAULT;
rc = ecore_rdma_add_user(dev->rdma_ctx, &oparams);
if (rc) {
@ -1044,20 +1017,18 @@ ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
QL_DPRINT12(ha, "Allocated user context %p\n",
&ctx->ibucontext);
return &ctx->ibucontext;
return (0);
err:
kfree(ctx);
return ERR_PTR(rc);
return (rc);
}
int
void
qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
{
struct qlnxr_ucontext *uctx = get_qlnxr_ucontext(ibctx);
struct qlnxr_dev *dev = uctx->dev;
qlnx_host_t *ha = dev->ha;
struct qlnxr_mm *mm, *tmp;
int status = 0;
QL_DPRINT12(ha, "Deallocating user context %p\n",
uctx);
@ -1073,8 +1044,6 @@ qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx)
list_del(&mm->entry);
kfree(mm);
}
kfree(uctx);
return status;
}
int
@ -1662,7 +1631,7 @@ err0:
}
int
qlnxr_dereg_mr(struct ib_mr *ib_mr)
qlnxr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
{
struct qlnxr_mr *mr = get_qlnxr_mr(ib_mr);
struct qlnxr_dev *dev = get_qlnxr_dev((ib_mr->device));
@ -1822,35 +1791,10 @@ err:
return rc;
}
#if __FreeBSD_version >= 1102000
struct ib_cq *
qlnxr_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata)
#else
#if __FreeBSD_version >= 1100000
struct ib_cq *
qlnxr_create_cq(struct ib_device *ibdev,
struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata)
#else
struct ib_cq *
qlnxr_create_cq(struct ib_device *ibdev,
int entries,
int vector,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata)
#endif /* #if __FreeBSD_version >= 1100000 */
#endif /* #if __FreeBSD_version >= 1102000 */
int
qlnxr_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata)
{
struct qlnxr_ucontext *ctx;
struct ecore_rdma_destroy_cq_out_params destroy_oparams;
@ -1863,13 +1807,13 @@ qlnxr_create_cq(struct ib_device *ibdev,
int vector = attr->comp_vector;
int entries = attr->cqe;
#endif
struct qlnxr_cq *cq;
struct qlnxr_cq *cq = get_qlnxr_cq(ibcq);
int chain_entries, rc, page_cnt;
u64 pbl_ptr;
u16 icid;
qlnx_host_t *ha;
dev = get_qlnxr_dev(ibdev);
dev = get_qlnxr_dev(ibcq->device);
ha = dev->ha;
QL_DPRINT12(ha, "called from %s. entries = %d, "
@ -1885,17 +1829,15 @@ qlnxr_create_cq(struct ib_device *ibdev,
"the number of entries %d is too high. "
"Must be equal or below %d.\n",
entries, QLNXR_MAX_CQES);
return ERR_PTR(-EINVAL);
return -EINVAL;
}
chain_entries = qlnxr_align_cq_entries(entries);
chain_entries = min_t(int, chain_entries, QLNXR_MAX_CQES);
cq = qlnx_zalloc((sizeof(struct qlnxr_cq)));
if (!cq)
return ERR_PTR(-ENOMEM);
if (udata) {
ctx = rdma_udata_to_drv_context(
udata, struct qlnxr_ucontext, ibucontext);
memset(&ureq, 0, sizeof(ureq));
if (ib_copy_from_udata(&ureq, udata,
@ -1911,13 +1853,15 @@ qlnxr_create_cq(struct ib_device *ibdev,
cq->cq_type = QLNXR_CQ_TYPE_USER;
qlnxr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr, ureq.len,
qlnxr_init_user_queue(&ctx->ibucontext, dev, &cq->q, ureq.addr, ureq.len,
IB_ACCESS_LOCAL_WRITE, 1, 1);
pbl_ptr = cq->q.pbl_tbl->pa;
page_cnt = cq->q.pbl_info.num_pbes;
cq->ibcq.cqe = chain_entries;
} else {
ctx = NULL;
cq->cq_type = QLNXR_CQ_TYPE_KERNEL;
rc = ecore_chain_alloc(&dev->ha->cdev,
@ -1944,8 +1888,7 @@ qlnxr_create_cq(struct ib_device *ibdev,
params.pbl_ptr = pbl_ptr;
params.pbl_two_level = 0;
if (ib_ctx != NULL) {
ctx = get_qlnxr_ucontext(ib_ctx);
if (udata) {
params.dpi = ctx->dpi;
} else {
params.dpi = dev->dpi;
@ -1959,7 +1902,7 @@ qlnxr_create_cq(struct ib_device *ibdev,
cq->sig = QLNXR_CQ_MAGIC_NUMBER;
spin_lock_init(&cq->cq_lock);
if (ib_ctx) {
if (udata) {
rc = qlnxr_copy_cq_uresp(dev, cq, udata);
if (rc)
goto err3;
@ -1990,7 +1933,7 @@ qlnxr_create_cq(struct ib_device *ibdev,
" number of entries = 0x%x\n",
cq->icid, cq, params.cq_size);
QL_DPRINT12(ha,"cq_addr = %p\n", cq);
return &cq->ibcq;
return (0);
err3:
destroy_iparams.icid = cq->icid;
@ -2004,11 +1947,9 @@ err1:
if (udata)
ib_umem_release(cq->q.umem);
err0:
kfree(cq);
QL_DPRINT12(ha, "exit error\n");
return ERR_PTR(-EINVAL);
return (-EINVAL);
}
int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
@ -2024,8 +1965,8 @@ int qlnxr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
return status;
}
int
qlnxr_destroy_cq(struct ib_cq *ibcq)
void
qlnxr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
{
struct qlnxr_dev *dev = get_qlnxr_dev((ibcq->device));
struct ecore_rdma_destroy_cq_out_params oparams;
@ -2055,25 +1996,21 @@ qlnxr_destroy_cq(struct ib_cq *ibcq)
if (rc) {
QL_DPRINT12(ha, "ecore_rdma_destroy_cq failed cq_id = %d\n",
cq->icid);
return rc;
return;
}
QL_DPRINT12(ha, "free cq->pbl cq_id = %d\n", cq->icid);
ecore_chain_free(&dev->ha->cdev, &cq->pbl);
}
if (ibcq->uobject && ibcq->uobject->context) {
if (udata) {
qlnxr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
ib_umem_release(cq->q.umem);
}
cq->sig = ~cq->sig;
kfree(cq);
QL_DPRINT12(ha, "exit cq_id = %d\n", cq->icid);
return rc;
}
static int
@ -2393,8 +2330,7 @@ qlnxr_set_common_qp_params(struct qlnxr_dev *dev,
}
static int
qlnxr_check_srq_params(struct ib_pd *ibpd,
struct qlnxr_dev *dev,
qlnxr_check_srq_params(struct qlnxr_dev *dev,
struct ib_srq_init_attr *attrs)
{
struct ecore_rdma_device *qattr;
@ -3281,8 +3217,6 @@ qlnxr_create_qp(struct ib_pd *ibpd,
return &qp->ibqp;
err:
kfree(qp);
QL_DPRINT12(ha, "failed exit\n");
return ERR_PTR(-EFAULT);
}
@ -3987,9 +3921,9 @@ qlnxr_cleanup_kernel(struct qlnxr_dev *dev, struct qlnxr_qp *qp)
return;
}
int
static int
qlnxr_free_qp_resources(struct qlnxr_dev *dev,
struct qlnxr_qp *qp)
struct qlnxr_qp *qp, struct ib_udata *udata)
{
int rc = 0;
qlnx_host_t *ha;
@ -4007,13 +3941,13 @@ qlnxr_free_qp_resources(struct qlnxr_dev *dev,
return rc;
}
if (qp->ibqp.uobject && qp->ibqp.uobject->context)
if (udata)
qlnxr_cleanup_user(dev, qp);
else
qlnxr_cleanup_kernel(dev, qp);
#endif
if (qp->ibqp.uobject && qp->ibqp.uobject->context)
if (udata)
qlnxr_cleanup_user(dev, qp);
else
qlnxr_cleanup_kernel(dev, qp);
@ -4030,7 +3964,7 @@ qlnxr_free_qp_resources(struct qlnxr_dev *dev,
}
int
qlnxr_destroy_qp(struct ib_qp *ibqp)
qlnxr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
{
struct qlnxr_qp *qp = get_qlnxr_qp(ibqp);
struct qlnxr_dev *dev = qp->dev;
@ -4060,12 +3994,11 @@ qlnxr_destroy_qp(struct ib_qp *ibqp)
qp->sig = ~qp->sig;
qlnxr_free_qp_resources(dev, qp);
qlnxr_free_qp_resources(dev, qp, udata);
if (atomic_dec_and_test(&qp->refcnt)) {
/* TODO: only for iWARP? */
qlnxr_idr_remove(dev, qp->qp_id);
kfree(qp);
}
QL_DPRINT12(ha, "exit\n");
@ -5837,7 +5770,8 @@ err0:
#if __FreeBSD_version >= 1102000
struct ib_mr *
qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, u32 max_num_sg)
qlnxr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
struct qlnxr_dev *dev;
struct qlnxr_mr *mr;
@ -6266,48 +6200,28 @@ err0:
#endif /* #if __FreeBSD_version >= 1102000 */
struct ib_ah *
#if __FreeBSD_version >= 1102000
qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
int
qlnxr_create_ah(struct ib_ah *ibah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata)
#else
qlnxr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr)
#endif /* #if __FreeBSD_version >= 1102000 */
{
struct qlnxr_dev *dev;
qlnx_host_t *ha;
struct qlnxr_ah *ah;
struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
dev = get_qlnxr_dev((ibpd->device));
dev = get_qlnxr_dev(ibah->device);
ha = dev->ha;
QL_DPRINT12(ha, "in create_ah\n");
ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
if (!ah) {
QL_DPRINT12(ha, "no address handle can be allocated\n");
return ERR_PTR(-ENOMEM);
}
ah->attr = *attr;
return &ah->ibah;
return (0);
}
int
qlnxr_destroy_ah(struct ib_ah *ibah)
void
qlnxr_destroy_ah(struct ib_ah *ibah, u32 flags)
{
struct qlnxr_dev *dev;
qlnx_host_t *ha;
struct qlnxr_ah *ah = get_qlnxr_ah(ibah);
dev = get_qlnxr_dev((ibah->device));
ha = dev->ha;
QL_DPRINT12(ha, "in destroy_ah\n");
kfree(ah);
return 0;
}
int
@ -7187,7 +7101,6 @@ qlnxr_iw_qp_rem_ref(struct ib_qp *ibqp)
if (atomic_dec_and_test(&qp->refcnt)) {
qlnxr_idr_remove(qp->dev, qp->qp_id);
kfree(qp);
}
QL_DPRINT12(ha, "exit \n");

62
sys/dev/qlnx/qlnxr/qlnxr_verbs.h
View File

@ -40,11 +40,12 @@ extern int qlnxr_query_gid(struct ib_device *,
int index,
union ib_gid *gid);
extern struct ib_srq *qlnxr_create_srq(struct ib_pd *,
extern int qlnxr_create_srq(struct ib_srq *ibsrq,
struct ib_srq_init_attr *,
struct ib_udata *);
extern int qlnxr_destroy_srq(struct ib_srq *);
extern void qlnxr_destroy_srq(struct ib_srq *,
struct ib_udata *);
extern int qlnxr_modify_srq(struct ib_srq *,
struct ib_srq_attr *,
@ -79,33 +80,15 @@ extern int qlnxr_modify_port(struct ib_device *,
extern enum rdma_link_layer qlnxr_link_layer(struct ib_device *device,
uint8_t port_num);
struct ib_pd *qlnxr_alloc_pd(struct ib_device *,
struct ib_ucontext *,
struct ib_udata *);
extern int qlnxr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *);
extern int qlnxr_dealloc_pd(struct ib_pd *pd);
extern void qlnxr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata);
#if __FreeBSD_version >= 1102000
extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
const struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata);
#else
#if __FreeBSD_version >= 1100000
extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
struct ib_cq_init_attr *attr,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata);
#else
extern struct ib_cq *qlnxr_create_cq(struct ib_device *ibdev,
int cqe,
int comp_vector,
struct ib_ucontext *ib_ctx,
struct ib_udata *udata);
#endif
#endif /* #if __FreeBSD_version >= 1102000 */
extern int qlnxr_create_cq(struct ib_cq *ibcq,
const struct ib_cq_init_attr *attr,
struct ib_udata *udata);
extern int qlnxr_destroy_cq(struct ib_cq *);
extern void qlnxr_destroy_cq(struct ib_cq *, struct ib_udata *);
extern int qlnxr_resize_cq(struct ib_cq *,
int cqe,
@ -129,22 +112,17 @@ extern int qlnxr_query_qp(struct ib_qp *,
int qp_attr_mask,
struct ib_qp_init_attr *);
extern int qlnxr_destroy_qp(struct ib_qp *);
extern int qlnxr_destroy_qp(struct ib_qp *, struct ib_udata *);
extern int qlnxr_query_pkey(struct ib_device *,
u8 port,
u16 index,
u16 *pkey);
#if __FreeBSD_version >= 1102000
extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd,
struct ib_ah_attr *attr, struct ib_udata *udata);
#else
extern struct ib_ah *qlnxr_create_ah(struct ib_pd *ibpd,
struct ib_ah_attr *attr);
#endif /* #if __FreeBSD_version >= 1102000 */
extern int qlnxr_destroy_ah(struct ib_ah *ibah);
extern int qlnxr_create_ah(struct ib_ah *ibah,
struct ib_ah_attr *attr, u32 flags,
struct ib_udata *udata);
extern void qlnxr_destroy_ah(struct ib_ah *ibah, u32 flags);
extern int qlnxr_query_ah(struct ib_ah *ibah,
struct ib_ah_attr *attr);
@ -195,7 +173,7 @@ extern struct ib_mr *qlnxr_reg_kernel_mr(struct ib_pd *,
u64 *iova_start);
#endif /* #if __FreeBSD_version < 1102000 */
extern int qlnxr_dereg_mr(struct ib_mr *);
extern int qlnxr_dereg_mr(struct ib_mr *, struct ib_udata *);
#if __FreeBSD_version >= 1102000
extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *,
@ -217,7 +195,9 @@ extern struct ib_mr *qlnxr_reg_user_mr(struct ib_pd *,
#if __FreeBSD_version >= 1102000
extern struct ib_mr *qlnxr_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type, u32 max_num_sg);
enum ib_mr_type mr_type, u32 max_num_sg,
struct ib_udata *udata);
extern int qlnxr_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg,
int sg_nents, unsigned int *sg_offset);
#else
@ -233,10 +213,10 @@ extern void qlnxr_free_frmr_page_list(struct ib_fast_reg_page_list *page_list);
#endif /* #if __FreeBSD_version >= 1102000 */
extern struct ib_ucontext *qlnxr_alloc_ucontext(struct ib_device *ibdev,
struct ib_udata *udata);
extern int qlnxr_alloc_ucontext(struct ib_ucontext *uctx,
struct ib_udata *udata);
extern int qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx);
extern void qlnxr_dealloc_ucontext(struct ib_ucontext *ibctx);
extern int qlnxr_mmap(struct ib_ucontext *, struct vm_area_struct *vma);

13
sys/modules/ibcore/Makefile
View File

@ -8,6 +8,7 @@ SRCS= \
ib_cache.c \
ib_cm.c \
ib_cma.c \
ib_core_uverbs.c \
ib_cq.c \
ib_device.c \
ib_fmr_pool.c \
@ -18,6 +19,7 @@ SRCS= \
ib_mad_rmpp.c \
ib_multicast.c \
ib_packer.c \
ib_rdma_core.c \
ib_roce_gid_mgmt.c \
ib_sa_query.c \
ib_smi.c \
@ -28,9 +30,20 @@ SRCS= \
ib_umem.c \
ib_user_mad.c \
ib_uverbs_cmd.c \
ib_uverbs_ioctl.c \
ib_uverbs_main.c \
ib_uverbs_marshall.c \
ib_uverbs_std_types.c \
ib_uverbs_std_types_async_fd.c \
ib_uverbs_std_types_counters.c \
ib_uverbs_std_types_cq.c \
ib_uverbs_std_types_device.c \
ib_uverbs_std_types_dm.c \
ib_uverbs_std_types_flow_action.c \
ib_uverbs_std_types_mr.c \
ib_uverbs_uapi.c \
ib_verbs.c
SRCS+= ${LINUXKPI_GENSRCS}
SRCS+= opt_inet.h opt_inet6.h

1
sys/modules/mlx5/Makefile
View File

@ -38,6 +38,7 @@ SRCS+= ${LINUXKPI_GENSRCS}
SRCS+= opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h
CFLAGS+= -I${SRCTOP}/sys/ofed/include
CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include
.if defined(CONFIG_BUILD_FPGA)

1
sys/modules/mlx5en/Makefile
View File

@ -29,6 +29,7 @@ CFLAGS+= -DCONFIG_MLX5_FPGA
.endif
CFLAGS+= -I${SRCTOP}/sys/ofed/include
CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include
.include <bsd.kmod.mk>

1
sys/modules/mlx5fpga_tools/Makefile
View File

@ -10,6 +10,7 @@ SRCS+= ${LINUXKPI_GENSRCS}
SRCS+= opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h
CFLAGS+= -I${SRCTOP}/sys/ofed/include
CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi
CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include
.include <bsd.kmod.mk>

1
sys/modules/mlx5ib/Makefile
View File

@ -6,6 +6,7 @@ SRCS= \
mlx5_ib_ah.c \
mlx5_ib_cong.c \
mlx5_ib_cq.c \
mlx5_ib_devx.c \
mlx5_ib_doorbell.c \
mlx5_ib_gsi.c \
mlx5_ib_mad.c \

1
sys/modules/mlxfw/Makefile
View File

@ -10,6 +10,7 @@ SRCS+= ${LINUXKPI_GENSRCS}
CFLAGS+= \
-I${SRCTOP}/sys/ofed/include \
-I${SRCTOP}/sys/ofed/include/uapi \
-I${SRCTOP}/sys/compat/linuxkpi/common/include \
-I${SRCTOP}/sys/contrib/xz-embedded/freebsd \
-I${SRCTOP}/sys/contrib/xz-embedded/linux/lib/xz

49
sys/ofed/drivers/infiniband/core/core_priv.h
View File

@ -44,6 +44,9 @@
#include <net/if_vlan_var.h>
/* Total number of ports combined across all struct ib_devices's */
#define RDMA_MAX_PORTS 8192
#ifdef CONFIG_INFINIBAND_ADDR_TRANS_CONFIGFS
int cma_configfs_init(void);
void cma_configfs_exit(void);
@ -128,6 +131,8 @@ int ib_cache_setup_one(struct ib_device *device);
void ib_cache_cleanup_one(struct ib_device *device);
void ib_cache_release_one(struct ib_device *device);
#define ib_rdmacg_try_charge(...) ({ 0; })
int addr_init(void);
void addr_cleanup(void);
@ -142,4 +147,48 @@ int ib_port_register_module_stat(struct ib_device *device, u8 port_num,
const char *name);
void ib_port_unregister_module_stat(struct kobject *kobj);
static inline struct ib_qp *_ib_create_qp(struct ib_device *dev,
struct ib_pd *pd,
struct ib_qp_init_attr *attr,
struct ib_udata *udata,
struct ib_uqp_object *uobj)
{
struct ib_qp *qp;
if (!dev->create_qp)
return ERR_PTR(-EOPNOTSUPP);
qp = dev->create_qp(pd, attr, udata);
if (IS_ERR(qp))
return qp;
qp->device = dev;
qp->pd = pd;
qp->uobject = uobj;
qp->real_qp = qp;
qp->qp_type = attr->qp_type;
qp->rwq_ind_tbl = attr->rwq_ind_tbl;
qp->send_cq = attr->send_cq;
qp->recv_cq = attr->recv_cq;
qp->srq = attr->srq;
qp->rwq_ind_tbl = attr->rwq_ind_tbl;
qp->event_handler = attr->event_handler;
atomic_set(&qp->usecnt, 0);
spin_lock_init(&qp->mr_lock);
return qp;
}
struct rdma_umap_priv {
struct vm_area_struct *vma;
struct list_head list;
struct rdma_user_mmap_entry *entry;
};
void rdma_umap_priv_init(struct rdma_umap_priv *priv,
struct vm_area_struct *vma,
struct rdma_user_mmap_entry *entry);
#endif /* _CORE_PRIV_H */

4
sys/ofed/drivers/infiniband/core/ib_agent.c
View File

@ -141,13 +141,13 @@ void agent_send_response(const struct ib_mad_hdr *mad_hdr, const struct ib_grh *
err2:
ib_free_send_mad(send_buf);
err1:
ib_destroy_ah(ah);
ib_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
}
static void agent_send_handler(struct ib_mad_agent *mad_agent,
struct ib_mad_send_wc *mad_send_wc)
{
ib_destroy_ah(mad_send_wc->send_buf->ah);
ib_destroy_ah(mad_send_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE);
ib_free_send_mad(mad_send_wc->send_buf);
}

6
sys/ofed/drivers/infiniband/core/ib_cm.c
View File

@ -351,7 +351,7 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
ret = -ENODEV;
goto out;
}
ah = ib_create_ah(mad_agent->qp->pd, &av->ah_attr);
ah = ib_create_ah(mad_agent->qp->pd, &av->ah_attr, 0);
if (IS_ERR(ah)) {
ret = PTR_ERR(ah);
goto out;
@ -363,7 +363,7 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
GFP_ATOMIC,
IB_MGMT_BASE_VERSION);
if (IS_ERR(m)) {
ib_destroy_ah(ah);
ib_destroy_ah(ah, 0);
ret = PTR_ERR(m);
goto out;
}
@ -408,7 +408,7 @@ static int cm_create_response_msg_ah(struct cm_port *port,
static void cm_free_msg(struct ib_mad_send_buf *msg)
{
if (msg->ah)
ib_destroy_ah(msg->ah);
ib_destroy_ah(msg->ah, 0);
if (msg->context[0])
cm_deref_id(msg->context[0]);
ib_free_send_mad(msg);

390
sys/ofed/drivers/infiniband/core/ib_core_uverbs.c Normal file
View File

@ -0,0 +1,390 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0
*
* Copyright (c) 2005 Mellanox Technologies. All rights reserved.
* Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved.
* Copyright 2019 Marvell. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <linux/xarray.h>
#include "uverbs.h"
#include "core_priv.h"
/**
* rdma_umap_priv_init() - Initialize the private data of a vma
*
* @priv: The already allocated private data
* @vma: The vm area struct that needs private data
* @entry: entry into the mmap_xa that needs to be linked with
* this vma
*
* Each time we map IO memory into user space this keeps track of the
* mapping. When the device is hot-unplugged we 'zap' the mmaps in user space
* to point to the zero page and allow the hot unplug to proceed.
*
* This is necessary for cases like PCI physical hot unplug as the actual BAR
* memory may vanish after this and access to it from userspace could MCE.
*
* RDMA drivers supporting disassociation must have their user space designed
* to cope in some way with their IO pages going to the zero page.
*
*/
void rdma_umap_priv_init(struct rdma_umap_priv *priv,
struct vm_area_struct *vma,
struct rdma_user_mmap_entry *entry)
{
struct ib_uverbs_file *ufile = vma->vm_file->private_data;
priv->vma = vma;
if (entry) {
kref_get(&entry->ref);
priv->entry = entry;
}
vma->vm_private_data = priv;
/* vm_ops is setup in ib_uverbs_mmap() to avoid module dependencies */
mutex_lock(&ufile->umap_lock);
list_add(&priv->list, &ufile->umaps);
mutex_unlock(&ufile->umap_lock);
}
EXPORT_SYMBOL(rdma_umap_priv_init);
/**
* rdma_user_mmap_io() - Map IO memory into a process
*
* @ucontext: associated user context
* @vma: the vma related to the current mmap call
* @pfn: pfn to map
* @size: size to map
* @prot: pgprot to use in remap call
* @entry: mmap_entry retrieved from rdma_user_mmap_entry_get(), or NULL
* if mmap_entry is not used by the driver
*
* This is to be called by drivers as part of their mmap() functions if they
* wish to send something like PCI-E BAR memory to userspace.
*
* Return -EINVAL on wrong flags or size, -EAGAIN on failure to map. 0 on
* success.
*/
int rdma_user_mmap_io(struct ib_ucontext *ucontext, struct vm_area_struct *vma,
unsigned long pfn, unsigned long size, pgprot_t prot,
struct rdma_user_mmap_entry *entry)
{
struct ib_uverbs_file *ufile = ucontext->ufile;
struct rdma_umap_priv *priv;
if (!(vma->vm_flags & VM_SHARED))
return -EINVAL;
if (vma->vm_end - vma->vm_start != size)
return -EINVAL;
/* Driver is using this wrong, must be called by ib_uverbs_mmap */
if (WARN_ON(!vma->vm_file ||
vma->vm_file->private_data != ufile))
return -EINVAL;
priv = kzalloc(sizeof(*priv), GFP_KERNEL);
if (!priv)
return -ENOMEM;
vma->vm_page_prot = prot;
if (io_remap_pfn_range(vma, vma->vm_start, pfn, size, prot)) {
kfree(priv);
return -EAGAIN;
}
rdma_umap_priv_init(priv, vma, entry);
return 0;
}
EXPORT_SYMBOL(rdma_user_mmap_io);
/**
* rdma_user_mmap_entry_get_pgoff() - Get an entry from the mmap_xa
*
* @ucontext: associated user context
* @pgoff: The mmap offset >> PAGE_SHIFT
*
* This function is called when a user tries to mmap with an offset (returned
* by rdma_user_mmap_get_offset()) it initially received from the driver. The
* rdma_user_mmap_entry was created by the function
* rdma_user_mmap_entry_insert(). This function increases the refcnt of the
* entry so that it won't be deleted from the xarray in the meantime.
*
* Return an reference to an entry if exists or NULL if there is no
* match. rdma_user_mmap_entry_put() must be called to put the reference.
*/
struct rdma_user_mmap_entry *
rdma_user_mmap_entry_get_pgoff(struct ib_ucontext *ucontext,
unsigned long pgoff)
{
struct rdma_user_mmap_entry *entry;
if (pgoff > U32_MAX)
return NULL;
xa_lock(&ucontext->mmap_xa);
entry = xa_load(&ucontext->mmap_xa, pgoff);
/*
* If refcount is zero, entry is already being deleted, driver_removed
* indicates that the no further mmaps are possible and we waiting for
* the active VMAs to be closed.
*/
if (!entry || entry->start_pgoff != pgoff || entry->driver_removed ||
!kref_get_unless_zero(&entry->ref))
goto err;
xa_unlock(&ucontext->mmap_xa);
return entry;
err:
xa_unlock(&ucontext->mmap_xa);
return NULL;
}
EXPORT_SYMBOL(rdma_user_mmap_entry_get_pgoff);
/**
* rdma_user_mmap_entry_get() - Get an entry from the mmap_xa
*
* @ucontext: associated user context
* @vma: the vma being mmap'd into
*
* This function is like rdma_user_mmap_entry_get_pgoff() except that it also
* checks that the VMA is correct.
*/
struct rdma_user_mmap_entry *
rdma_user_mmap_entry_get(struct ib_ucontext *ucontext,
struct vm_area_struct *vma)
{
struct rdma_user_mmap_entry *entry;
if (!(vma->vm_flags & VM_SHARED))
return NULL;
entry = rdma_user_mmap_entry_get_pgoff(ucontext, vma->vm_pgoff);
if (!entry)
return NULL;
if (entry->npages * PAGE_SIZE != vma->vm_end - vma->vm_start) {
rdma_user_mmap_entry_put(entry);
return NULL;
}
return entry;
}
EXPORT_SYMBOL(rdma_user_mmap_entry_get);
static void rdma_user_mmap_entry_free(struct kref *kref)
{
struct rdma_user_mmap_entry *entry =
container_of(kref, struct rdma_user_mmap_entry, ref);
struct ib_ucontext *ucontext = entry->ucontext;
unsigned long i;
/*
* Erase all entries occupied by this single entry, this is deferred
* until all VMA are closed so that the mmap offsets remain unique.
*/
xa_lock(&ucontext->mmap_xa);
for (i = 0; i < entry->npages; i++)
__xa_erase(&ucontext->mmap_xa, entry->start_pgoff + i);
xa_unlock(&ucontext->mmap_xa);
if (ucontext->device->mmap_free)
ucontext->device->mmap_free(entry);
}
/**
* rdma_user_mmap_entry_put() - Drop reference to the mmap entry
*
* @entry: an entry in the mmap_xa
*
* This function is called when the mapping is closed if it was
* an io mapping or when the driver is done with the entry for
* some other reason.
* Should be called after rdma_user_mmap_entry_get was called
* and entry is no longer needed. This function will erase the
* entry and free it if its refcnt reaches zero.
*/
void rdma_user_mmap_entry_put(struct rdma_user_mmap_entry *entry)
{
kref_put(&entry->ref, rdma_user_mmap_entry_free);
}
EXPORT_SYMBOL(rdma_user_mmap_entry_put);
/**
* rdma_user_mmap_entry_remove() - Drop reference to entry and
* mark it as unmmapable
*
* @entry: the entry to insert into the mmap_xa
*
* Drivers can call this to prevent userspace from creating more mappings for
* entry, however existing mmaps continue to exist and ops->mmap_free() will
* not be called until all user mmaps are destroyed.
*/
void rdma_user_mmap_entry_remove(struct rdma_user_mmap_entry *entry)
{
if (!entry)
return;
xa_lock(&entry->ucontext->mmap_xa);
entry->driver_removed = true;
xa_unlock(&entry->ucontext->mmap_xa);
kref_put(&entry->ref, rdma_user_mmap_entry_free);
}
EXPORT_SYMBOL(rdma_user_mmap_entry_remove);
/**
* rdma_user_mmap_entry_insert_range() - Insert an entry to the mmap_xa
* in a given range.
*
* @ucontext: associated user context.
* @entry: the entry to insert into the mmap_xa
* @length: length of the address that will be mmapped
* @min_pgoff: minimum pgoff to be returned
* @max_pgoff: maximum pgoff to be returned
*
* This function should be called by drivers that use the rdma_user_mmap
* interface for implementing their mmap syscall A database of mmap offsets is
* handled in the core and helper functions are provided to insert entries
* into the database and extract entries when the user calls mmap with the
* given offset. The function allocates a unique page offset in a given range
* that should be provided to user, the user will use the offset to retrieve
* information such as address to be mapped and how.
*
* Return: 0 on success and -ENOMEM on failure
*/
int rdma_user_mmap_entry_insert_range(struct ib_ucontext *ucontext,
struct rdma_user_mmap_entry *entry,
size_t length, u32 min_pgoff,
u32 max_pgoff)
{
struct ib_uverbs_file *ufile = ucontext->ufile;
u32 xa_first, xa_last, npages;
int err;
u32 i;
u32 j;
if (!entry)
return -EINVAL;
kref_init(&entry->ref);
entry->ucontext = ucontext;
/*
* We want the whole allocation to be done without interruption from a
* different thread. The allocation requires finding a free range and
* storing. During the xa_insert the lock could be released, possibly
* allowing another thread to choose the same range.
*/
mutex_lock(&ufile->umap_lock);
xa_lock(&ucontext->mmap_xa);
/* We want to find an empty range */
npages = (u32)DIV_ROUND_UP(length, PAGE_SIZE);
entry->npages = npages;
/* Find an empty range */
for (i = min_pgoff, j = 0; (i + j) <= max_pgoff && j != npages; ) {
if (xa_load(&ucontext->mmap_xa, i + j) != NULL) {
if (unlikely(i + j == max_pgoff))
break;
i = i + j + 1;
j = 0;
} else {
if (unlikely(i + j == max_pgoff))
break;
j++;
}
}
if (j != npages)
goto err_unlock;
xa_first = i;
xa_last = i + j;
for (i = xa_first; i < xa_last; i++) {
err = __xa_insert(&ucontext->mmap_xa, i, entry, GFP_KERNEL);
if (err)
goto err_undo;
}
/*
* Internally the kernel uses a page offset, in libc this is a byte
* offset. Drivers should not return pgoff to userspace.
*/
entry->start_pgoff = xa_first;
xa_unlock(&ucontext->mmap_xa);
mutex_unlock(&ufile->umap_lock);
return 0;
err_undo:
for (; i > xa_first; i--)
__xa_erase(&ucontext->mmap_xa, i - 1);
err_unlock:
xa_unlock(&ucontext->mmap_xa);
mutex_unlock(&ufile->umap_lock);
return -ENOMEM;
}
EXPORT_SYMBOL(rdma_user_mmap_entry_insert_range);
/**
* rdma_user_mmap_entry_insert() - Insert an entry to the mmap_xa.
*
* @ucontext: associated user context.
* @entry: the entry to insert into the mmap_xa
* @length: length of the address that will be mmapped
*
* This function should be called by drivers that use the rdma_user_mmap
* interface for handling user mmapped addresses. The database is handled in
* the core and helper functions are provided to insert entries into the
* database and extract entries when the user calls mmap with the given offset.
* The function allocates a unique page offset that should be provided to user,
* the user will use the offset to retrieve information such as address to
* be mapped and how.
*
* Return: 0 on success and -ENOMEM on failure
*/
int rdma_user_mmap_entry_insert(struct ib_ucontext *ucontext,
struct rdma_user_mmap_entry *entry,
size_t length)
{
return rdma_user_mmap_entry_insert_range(ucontext, entry, length, 0,
U32_MAX);
}
EXPORT_SYMBOL(rdma_user_mmap_entry_insert);

32
sys/ofed/drivers/infiniband/core/ib_cq.c
View File

@ -86,14 +86,17 @@ ib_cq_completion_workqueue(struct ib_cq *cq, void *private)
}
struct ib_cq *
ib_alloc_cq(struct ib_device *dev, void *private,
int nr_cqe, int comp_vector, enum ib_poll_context poll_ctx)
__ib_alloc_cq_user(struct ib_device *dev, void *private,
int nr_cqe, int comp_vector,
enum ib_poll_context poll_ctx,
const char *caller, struct ib_udata *udata)
{
struct ib_cq_init_attr cq_attr = {
.cqe = nr_cqe,
.comp_vector = comp_vector,
};
struct ib_cq *cq;
int ret;
/*
* Check for invalid parameters early on to avoid
@ -108,17 +111,19 @@ ib_alloc_cq(struct ib_device *dev, void *private,
return (ERR_PTR(-EINVAL));
}
cq = dev->create_cq(dev, &cq_attr, NULL, NULL);
if (IS_ERR(cq))
return (cq);
cq = rdma_zalloc_drv_obj(dev, ib_cq);
if (!cq)
return ERR_PTR(-ENOMEM);
cq->device = dev;
cq->uobject = NULL;
cq->event_handler = NULL;
cq->cq_context = private;
cq->poll_ctx = poll_ctx;
atomic_set(&cq->usecnt, 0);
ret = dev->create_cq(cq, &cq_attr, NULL);
if (ret)
goto out_free_cq;
switch (poll_ctx) {
case IB_POLL_DIRECT:
cq->comp_handler = NULL; /* no hardware completions */
@ -133,11 +138,15 @@ ib_alloc_cq(struct ib_device *dev, void *private,
break;
}
return (cq);
out_free_cq:
kfree(cq);
return (ERR_PTR(ret));
}
EXPORT_SYMBOL(ib_alloc_cq);
EXPORT_SYMBOL(__ib_alloc_cq_user);
void
ib_free_cq(struct ib_cq *cq)
ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
{
if (WARN_ON_ONCE(atomic_read(&cq->usecnt) != 0))
@ -154,6 +163,7 @@ ib_free_cq(struct ib_cq *cq)
break;
}
(void)cq->device->destroy_cq(cq);
cq->device->destroy_cq(cq, udata);
kfree(cq);
}
EXPORT_SYMBOL(ib_free_cq);
EXPORT_SYMBOL(ib_free_cq_user);

10
sys/ofed/drivers/infiniband/core/ib_mad_rmpp.c
View File

@ -86,7 +86,7 @@ static void destroy_rmpp_recv(struct mad_rmpp_recv *rmpp_recv)
{
deref_rmpp_recv(rmpp_recv);
wait_for_completion(&rmpp_recv->comp);
ib_destroy_ah(rmpp_recv->ah);
ib_destroy_ah(rmpp_recv->ah, RDMA_DESTROY_AH_SLEEPABLE);
kfree(rmpp_recv);
}
@ -176,7 +176,7 @@ static struct ib_mad_send_buf *alloc_response_msg(struct ib_mad_agent *agent,
hdr_len, 0, GFP_KERNEL,
IB_MGMT_BASE_VERSION);
if (IS_ERR(msg))
ib_destroy_ah(ah);
ib_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
else {
msg->ah = ah;
msg->context[0] = ah;
@ -206,7 +206,7 @@ static void ack_ds_ack(struct ib_mad_agent_private *agent,
ret = ib_post_send_mad(msg, NULL);
if (ret) {
ib_destroy_ah(msg->ah);
ib_destroy_ah(msg->ah, RDMA_DESTROY_AH_SLEEPABLE);
ib_free_send_mad(msg);
}
}
@ -214,7 +214,7 @@ static void ack_ds_ack(struct ib_mad_agent_private *agent,
void ib_rmpp_send_handler(struct ib_mad_send_wc *mad_send_wc)
{
if (mad_send_wc->send_buf->context[0] == mad_send_wc->send_buf->ah)
ib_destroy_ah(mad_send_wc->send_buf->ah);
ib_destroy_ah(mad_send_wc->send_buf->ah, RDMA_DESTROY_AH_SLEEPABLE);
ib_free_send_mad(mad_send_wc->send_buf);
}
@ -242,7 +242,7 @@ static void nack_recv(struct ib_mad_agent_private *agent,
ret = ib_post_send_mad(msg, NULL);
if (ret) {
ib_destroy_ah(msg->ah);
ib_destroy_ah(msg->ah, RDMA_DESTROY_AH_SLEEPABLE);
ib_free_send_mad(msg);
}
}

943
sys/ofed/drivers/infiniband/core/ib_rdma_core.c Normal file
View File

@ -0,0 +1,943 @@
/*
* Copyright (c) 2016, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <asm/atomic.h>
#include <linux/file.h>
#include <linux/lockdep.h>
#include <rdma/ib_verbs.h>
#include <rdma/uverbs_types.h>
#include <linux/rcupdate.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/rdma_user_ioctl.h>
#include "uverbs.h"
#include "core_priv.h"
#include "rdma_core.h"
static void uverbs_uobject_free(struct kref *ref)
{
kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu);
}
/*
* In order to indicate we no longer needs this uobject, uverbs_uobject_put
* is called. When the reference count is decreased, the uobject is freed.
* For example, this is used when attaching a completion channel to a CQ.
*/
void uverbs_uobject_put(struct ib_uobject *uobject)
{
kref_put(&uobject->ref, uverbs_uobject_free);
}
EXPORT_SYMBOL(uverbs_uobject_put);
static int uverbs_try_lock_object(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
{
/*
* When a shared access is required, we use a positive counter. Each
* shared access request checks that the value != -1 and increment it.
* Exclusive access is required for operations like write or destroy.
* In exclusive access mode, we check that the counter is zero (nobody
* claimed this object) and we set it to -1. Releasing a shared access
* lock is done simply by decreasing the counter. As for exclusive
* access locks, since only a single one of them is is allowed
* concurrently, setting the counter to zero is enough for releasing
* this lock.
*/
switch (mode) {
case UVERBS_LOOKUP_READ:
return atomic_fetch_add_unless(&uobj->usecnt, 1, -1) == -1 ?
-EBUSY : 0;
case UVERBS_LOOKUP_WRITE:
/* lock is exclusive */
return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY;
case UVERBS_LOOKUP_DESTROY:
return 0;
}
return 0;
}
static void assert_uverbs_usecnt(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
{
#ifdef CONFIG_LOCKDEP
switch (mode) {
case UVERBS_LOOKUP_READ:
WARN_ON(atomic_read(&uobj->usecnt) <= 0);
break;
case UVERBS_LOOKUP_WRITE:
WARN_ON(atomic_read(&uobj->usecnt) != -1);
break;
case UVERBS_LOOKUP_DESTROY:
break;
}
#endif
}
/*
* This must be called with the hw_destroy_rwsem locked for read or write,
* also the uobject itself must be locked for write.
*
* Upon return the HW object is guaranteed to be destroyed.
*
* For RDMA_REMOVE_ABORT, the hw_destroy_rwsem is not required to be held,
* however the type's allocat_commit function cannot have been called and the
* uobject cannot be on the uobjects_lists
*
* For RDMA_REMOVE_DESTROY the caller shold be holding a kref (eg via
* rdma_lookup_get_uobject) and the object is left in a state where the caller
* needs to call rdma_lookup_put_uobject.
*
* For all other destroy modes this function internally unlocks the uobject
* and consumes the kref on the uobj.
*/
static int uverbs_destroy_uobject(struct ib_uobject *uobj,
enum rdma_remove_reason reason,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = attrs->ufile;
unsigned long flags;
int ret;
lockdep_assert_held(&ufile->hw_destroy_rwsem);
assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);
if (reason == RDMA_REMOVE_ABORT) {
WARN_ON(!list_empty(&uobj->list));
WARN_ON(!uobj->context);
uobj->uapi_object->type_class->alloc_abort(uobj);
} else if (uobj->object) {
ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
attrs);
if (ret) {
if (ib_is_destroy_retryable(ret, reason, uobj))
return ret;
/* Nothing to be done, dangle the memory and move on */
WARN(true,
"ib_uverbs: failed to remove uobject id %d, driver err=%d",
uobj->id, ret);
}
uobj->object = NULL;
}
uobj->context = NULL;
/*
* For DESTROY the usecnt is held write locked, the caller is expected
* to put it unlock and put the object when done with it. Only DESTROY
* can remove the IDR handle.
*/
if (reason != RDMA_REMOVE_DESTROY)
atomic_set(&uobj->usecnt, 0);
else
uobj->uapi_object->type_class->remove_handle(uobj);
if (!list_empty(&uobj->list)) {
spin_lock_irqsave(&ufile->uobjects_lock, flags);
list_del_init(&uobj->list);
spin_unlock_irqrestore(&ufile->uobjects_lock, flags);
/*
* Pairs with the get in rdma_alloc_commit_uobject(), could
* destroy uobj.
*/
uverbs_uobject_put(uobj);
}
/*
* When aborting the stack kref remains owned by the core code, and is
* not transferred into the type. Pairs with the get in alloc_uobj
*/
if (reason == RDMA_REMOVE_ABORT)
uverbs_uobject_put(uobj);
return 0;
}
/*
* This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY
* sequence. It should only be used from command callbacks. On success the
* caller must pair this with rdma_lookup_put_uobject(LOOKUP_WRITE). This
* version requires the caller to have already obtained an
* LOOKUP_DESTROY uobject kref.
*/
int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = attrs->ufile;
int ret;
down_read(&ufile->hw_destroy_rwsem);
ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE);
if (ret)
goto out_unlock;
ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY, attrs);
if (ret) {
atomic_set(&uobj->usecnt, 0);
goto out_unlock;
}
out_unlock:
up_read(&ufile->hw_destroy_rwsem);
return ret;
}
/*
* uobj_get_destroy destroys the HW object and returns a handle to the uobj
* with a NULL object pointer. The caller must pair this with
* uverbs_put_destroy.
*/
struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
u32 id, struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj;
int ret;
uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id,
UVERBS_LOOKUP_DESTROY, attrs);
if (IS_ERR(uobj))
return uobj;
ret = uobj_destroy(uobj, attrs);
if (ret) {
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
return ERR_PTR(ret);
}
return uobj;
}
/*
* Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success
* (negative errno on failure). For use by callers that do not need the uobj.
*/
int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj;
uobj = __uobj_get_destroy(obj, id, attrs);
if (IS_ERR(uobj))
return PTR_ERR(uobj);
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
return 0;
}
/* alloc_uobj must be undone by uverbs_destroy_uobject() */
static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs,
const struct uverbs_api_object *obj)
{
struct ib_uverbs_file *ufile = attrs->ufile;
struct ib_uobject *uobj;
if (!attrs->context) {
struct ib_ucontext *ucontext =
ib_uverbs_get_ucontext_file(ufile);
if (IS_ERR(ucontext))
return ERR_CAST(ucontext);
attrs->context = ucontext;
}
uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL);
if (!uobj)
return ERR_PTR(-ENOMEM);
/*
* user_handle should be filled by the handler,
* The object is added to the list in the commit stage.
*/
uobj->ufile = ufile;
uobj->context = attrs->context;
INIT_LIST_HEAD(&uobj->list);
uobj->uapi_object = obj;
/*
* Allocated objects start out as write locked to deny any other
* syscalls from accessing them until they are committed. See
* rdma_alloc_commit_uobject
*/
atomic_set(&uobj->usecnt, -1);
kref_init(&uobj->ref);
return uobj;
}
#define NULL_IB_UOBJECT ((struct ib_uobject *)1)
static int idr_add_uobj(struct ib_uobject *uobj)
{
/*
* We start with allocating an idr pointing to NULL. This represents an
* object which isn't initialized yet. We'll replace it later on with
* the real object once we commit.
*/
return xa_alloc(&uobj->ufile->idr, &uobj->id, NULL_IB_UOBJECT, xa_limit_32b,
GFP_KERNEL);
}
/* Returns the ib_uobject or an error. The caller should check for IS_ERR. */
static struct ib_uobject *
lookup_get_idr_uobject(const struct uverbs_api_object *obj,
struct ib_uverbs_file *ufile, s64 id,
enum rdma_lookup_mode mode)
{
struct ib_uobject *uobj;
if (id < 0 || id > ULONG_MAX)
return ERR_PTR(-EINVAL);
rcu_read_lock();
/*
* The idr_find is guaranteed to return a pointer to something that
* isn't freed yet, or NULL, as the free after idr_remove goes through
* kfree_rcu(). However the object may still have been released and
* kfree() could be called at any time.
*/
uobj = xa_load(&ufile->idr, id);
if (!uobj || uobj == NULL_IB_UOBJECT || !kref_get_unless_zero(&uobj->ref))
uobj = ERR_PTR(-ENOENT);
rcu_read_unlock();
return uobj;
}
static struct ib_uobject *
lookup_get_fd_uobject(const struct uverbs_api_object *obj,
struct ib_uverbs_file *ufile, s64 id,
enum rdma_lookup_mode mode)
{
const struct uverbs_obj_fd_type *fd_type;
struct file *f;
struct ib_uobject *uobject;
int fdno = id;
if (fdno != id)
return ERR_PTR(-EINVAL);
if (mode != UVERBS_LOOKUP_READ)
return ERR_PTR(-EOPNOTSUPP);
if (!obj->type_attrs)
return ERR_PTR(-EIO);
fd_type =
container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
f = fget(fdno);
if (!f)
return ERR_PTR(-EBADF);
uobject = f->private_data;
/*
* fget(id) ensures we are not currently running
* uverbs_uobject_fd_release(), and the caller is expected to ensure
* that release is never done while a call to lookup is possible.
*/
if (f->f_op != fd_type->fops) {
fput(f);
return ERR_PTR(-EBADF);
}
uverbs_uobject_get(uobject);
return uobject;
}
struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
struct ib_uverbs_file *ufile, s64 id,
enum rdma_lookup_mode mode,
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj;
int ret;
if (obj == ERR_PTR(-ENOMSG)) {
/* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */
uobj = lookup_get_idr_uobject(NULL, ufile, id, mode);
if (IS_ERR(uobj))
return uobj;
} else {
if (IS_ERR(obj))
return ERR_PTR(-EINVAL);
uobj = obj->type_class->lookup_get(obj, ufile, id, mode);
if (IS_ERR(uobj))
return uobj;
if (uobj->uapi_object != obj) {
ret = -EINVAL;
goto free;
}
}
/*
* If we have been disassociated block every command except for
* DESTROY based commands.
*/
if (mode != UVERBS_LOOKUP_DESTROY &&
!srcu_dereference(ufile->device->ib_dev,
&ufile->device->disassociate_srcu)) {
ret = -EIO;
goto free;
}
ret = uverbs_try_lock_object(uobj, mode);
if (ret)
goto free;
if (attrs)
attrs->context = uobj->context;
return uobj;
free:
uobj->uapi_object->type_class->lookup_put(uobj, mode);
uverbs_uobject_put(uobj);
return ERR_PTR(ret);
}
static struct ib_uobject *
alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs)
{
int ret;
struct ib_uobject *uobj;
uobj = alloc_uobj(attrs, obj);
if (IS_ERR(uobj))
return uobj;
ret = idr_add_uobj(uobj);
if (ret)
goto uobj_put;
ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device,
RDMACG_RESOURCE_HCA_OBJECT);
if (ret)
goto remove;
return uobj;
remove:
xa_erase(&attrs->ufile->idr, uobj->id);
uobj_put:
uverbs_uobject_put(uobj);
return ERR_PTR(ret);
}
static struct ib_uobject *
alloc_begin_fd_uobject(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs)
{
const struct uverbs_obj_fd_type *fd_type =
container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
int new_fd;
struct ib_uobject *uobj;
struct file *filp;
if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release))
return ERR_PTR(-EINVAL);
new_fd = get_unused_fd_flags(O_CLOEXEC);
if (new_fd < 0)
return ERR_PTR(new_fd);
uobj = alloc_uobj(attrs, obj);
if (IS_ERR(uobj))
goto err_fd;
/* Note that uverbs_uobject_fd_release() is called during abort */
filp = alloc_file(fd_type->flags, fd_type->fops);
if (IS_ERR(filp)) {
uobj = ERR_CAST(filp);
goto err_uobj;
}
uobj->object = filp;
uobj->id = new_fd;
return uobj;
err_uobj:
uverbs_uobject_put(uobj);
err_fd:
put_unused_fd(new_fd);
return uobj;
}
struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = attrs->ufile;
struct ib_uobject *ret;
if (IS_ERR(obj))
return ERR_PTR(-EINVAL);
/*
* The hw_destroy_rwsem is held across the entire object creation and
* released during rdma_alloc_commit_uobject or
* rdma_alloc_abort_uobject
*/
if (!down_read_trylock(&ufile->hw_destroy_rwsem))
return ERR_PTR(-EIO);
ret = obj->type_class->alloc_begin(obj, attrs);
if (IS_ERR(ret)) {
up_read(&ufile->hw_destroy_rwsem);
return ret;
}
return ret;
}
static void alloc_abort_idr_uobject(struct ib_uobject *uobj)
{
xa_erase(&uobj->ufile->idr, uobj->id);
}
static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
const struct uverbs_obj_idr_type *idr_type =
container_of(uobj->uapi_object->type_attrs,
struct uverbs_obj_idr_type, type);
int ret = idr_type->destroy_object(uobj, why, attrs);
/*
* We can only fail gracefully if the user requested to destroy the
* object or when a retry may be called upon an error.
* In the rest of the cases, just remove whatever you can.
*/
if (ib_is_destroy_retryable(ret, why, uobj))
return ret;
if (why == RDMA_REMOVE_ABORT)
return 0;
return 0;
}
static void remove_handle_idr_uobject(struct ib_uobject *uobj)
{
xa_erase(&uobj->ufile->idr, uobj->id);
/* Matches the kref in alloc_commit_idr_uobject */
uverbs_uobject_put(uobj);
}
static void alloc_abort_fd_uobject(struct ib_uobject *uobj)
{
struct file *filp = uobj->object;
fput(filp);
put_unused_fd(uobj->id);
}
static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
const struct uverbs_obj_fd_type *fd_type = container_of(
uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type);
int ret = fd_type->destroy_object(uobj, why);
if (ib_is_destroy_retryable(ret, why, uobj))
return ret;
return 0;
}
static void remove_handle_fd_uobject(struct ib_uobject *uobj)
{
}
static void alloc_commit_idr_uobject(struct ib_uobject *uobj)
{
struct ib_uverbs_file *ufile = uobj->ufile;
void *old;
/*
* We already allocated this IDR with a NULL object, so
* this shouldn't fail.
*
* NOTE: Storing the uobj transfers our kref on uobj to the XArray.
* It will be put by remove_commit_idr_uobject()
*/
old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL);
WARN_ON(old != NULL_IB_UOBJECT);
}
static void alloc_commit_fd_uobject(struct ib_uobject *uobj)
{
int fd = uobj->id;
struct file *filp = uobj->object;
/* Matching put will be done in uverbs_uobject_fd_release() */
kref_get(&uobj->ufile->ref);
/* This shouldn't be used anymore. Use the file object instead */
uobj->id = 0;
/*
* NOTE: Once we install the file we loose ownership of our kref on
* uobj. It will be put by uverbs_uobject_fd_release()
*/
filp->private_data = uobj;
fd_install(fd, filp);
}
/*
* In all cases rdma_alloc_commit_uobject() consumes the kref to uobj and the
* caller can no longer assume uobj is valid. If this function fails it
* destroys the uboject, including the attached HW object.
*/
void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = attrs->ufile;
/* alloc_commit consumes the uobj kref */
uobj->uapi_object->type_class->alloc_commit(uobj);
/* kref is held so long as the uobj is on the uobj list. */
uverbs_uobject_get(uobj);
spin_lock_irq(&ufile->uobjects_lock);
list_add(&uobj->list, &ufile->uobjects);
spin_unlock_irq(&ufile->uobjects_lock);
/* matches atomic_set(-1) in alloc_uobj */
atomic_set(&uobj->usecnt, 0);
/* Matches the down_read in rdma_alloc_begin_uobject */
up_read(&ufile->hw_destroy_rwsem);
}
/*
* This consumes the kref for uobj. It is up to the caller to unwind the HW
* object and anything else connected to uobj before calling this.
*/
void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_file *ufile = uobj->ufile;
uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
/* Matches the down_read in rdma_alloc_begin_uobject */
up_read(&ufile->hw_destroy_rwsem);
}
static void lookup_put_idr_uobject(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
{
}
static void lookup_put_fd_uobject(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
{
struct file *filp = uobj->object;
WARN_ON(mode != UVERBS_LOOKUP_READ);
/*
* This indirectly calls uverbs_uobject_fd_release() and free the
* object
*/
fput(filp);
}
void rdma_lookup_put_uobject(struct ib_uobject *uobj,
enum rdma_lookup_mode mode)
{
assert_uverbs_usecnt(uobj, mode);
uobj->uapi_object->type_class->lookup_put(uobj, mode);
/*
* In order to unlock an object, either decrease its usecnt for
* read access or zero it in case of exclusive access. See
* uverbs_try_lock_object for locking schema information.
*/
switch (mode) {
case UVERBS_LOOKUP_READ:
atomic_dec(&uobj->usecnt);
break;
case UVERBS_LOOKUP_WRITE:
atomic_set(&uobj->usecnt, 0);
break;
case UVERBS_LOOKUP_DESTROY:
break;
}
/* Pairs with the kref obtained by type->lookup_get */
uverbs_uobject_put(uobj);
}
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile)
{
xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC);
}
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
{
struct ib_uobject *entry;
unsigned long id;
/*
* At this point uverbs_cleanup_ufile() is guaranteed to have run, and
* there are no HW objects left, however the xarray is still populated
* with anything that has not been cleaned up by userspace. Since the
* kref on ufile is 0, nothing is allowed to call lookup_get.
*
* This is an optimized equivalent to remove_handle_idr_uobject
*/
xa_for_each(&ufile->idr, id, entry) {
WARN_ON(entry->object);
uverbs_uobject_put(entry);
}
xa_destroy(&ufile->idr);
}
const struct uverbs_obj_type_class uverbs_idr_class = {
.alloc_begin = alloc_begin_idr_uobject,
.lookup_get = lookup_get_idr_uobject,
.alloc_commit = alloc_commit_idr_uobject,
.alloc_abort = alloc_abort_idr_uobject,
.lookup_put = lookup_put_idr_uobject,
.destroy_hw = destroy_hw_idr_uobject,
.remove_handle = remove_handle_idr_uobject,
};
EXPORT_SYMBOL(uverbs_idr_class);
/*
* Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct
* file_operations release method.
*/
int uverbs_uobject_fd_release(struct inode *inode, struct file *filp)
{
struct ib_uverbs_file *ufile;
struct ib_uobject *uobj;
/*
* This can only happen if the fput came from alloc_abort_fd_uobject()
*/
if (!filp->private_data)
return 0;
uobj = filp->private_data;
ufile = uobj->ufile;
if (down_read_trylock(&ufile->hw_destroy_rwsem)) {
struct uverbs_attr_bundle attrs = {
.context = uobj->context,
.ufile = ufile,
};
/*
* lookup_get_fd_uobject holds the kref on the struct file any
* time a FD uobj is locked, which prevents this release
* method from being invoked. Meaning we can always get the
* write lock here, or we have a kernel bug.
*/
WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE));
uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE, &attrs);
up_read(&ufile->hw_destroy_rwsem);
}
/* Matches the get in alloc_commit_fd_uobject() */
kref_put(&ufile->ref, ib_uverbs_release_file);
/* Pairs with filp->private_data in alloc_begin_fd_uobject */
uverbs_uobject_put(uobj);
return 0;
}
EXPORT_SYMBOL(uverbs_uobject_fd_release);
/*
* Drop the ucontext off the ufile and completely disconnect it from the
* ib_device
*/
static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
struct ib_ucontext *ucontext = ufile->ucontext;
struct ib_device *ib_dev = ucontext->device;
/*
* If we are closing the FD then the user mmap VMAs must have
* already been destroyed as they hold on to the filep, otherwise
* they need to be zap'd.
*/
if (reason == RDMA_REMOVE_DRIVER_REMOVE) {
uverbs_user_mmap_disassociate(ufile);
if (ib_dev->disassociate_ucontext)
ib_dev->disassociate_ucontext(ucontext);
}
ib_dev->dealloc_ucontext(ucontext);
WARN_ON(!xa_empty(&ucontext->mmap_xa));
kfree(ucontext);
ufile->ucontext = NULL;
}
static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
struct ib_uobject *obj, *next_obj;
int ret = -EINVAL;
struct uverbs_attr_bundle attrs = { .ufile = ufile };
/*
* This shouldn't run while executing other commands on this
* context. Thus, the only thing we should take care of is
* releasing a FD while traversing this list. The FD could be
* closed and released from the _release fop of this FD.
* In order to mitigate this, we add a lock.
* We take and release the lock per traversal in order to let
* other threads (which might still use the FDs) chance to run.
*/
list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) {
attrs.context = obj->context;
/*
* if we hit this WARN_ON, that means we are
* racing with a lookup_get.
*/
WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));
if (!uverbs_destroy_uobject(obj, reason, &attrs))
ret = 0;
else
atomic_set(&obj->usecnt, 0);
}
return ret;
}
/*
* Destroy the uncontext and every uobject associated with it.
*
* This is internally locked and can be called in parallel from multiple
* contexts.
*/
void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason)
{
down_write(&ufile->hw_destroy_rwsem);
/*
* If a ucontext was never created then we can't have any uobjects to
* cleanup, nothing to do.
*/
if (!ufile->ucontext)
goto done;
ufile->ucontext->closing = true;
ufile->ucontext->cleanup_retryable = true;
while (!list_empty(&ufile->uobjects))
if (__uverbs_cleanup_ufile(ufile, reason)) {
/*
* No entry was cleaned-up successfully during this
* iteration
*/
break;
}
ufile->ucontext->cleanup_retryable = false;
if (!list_empty(&ufile->uobjects))
__uverbs_cleanup_ufile(ufile, reason);
ufile_destroy_ucontext(ufile, reason);
done:
up_write(&ufile->hw_destroy_rwsem);
}
const struct uverbs_obj_type_class uverbs_fd_class = {
.alloc_begin = alloc_begin_fd_uobject,
.lookup_get = lookup_get_fd_uobject,
.alloc_commit = alloc_commit_fd_uobject,
.alloc_abort = alloc_abort_fd_uobject,
.lookup_put = lookup_put_fd_uobject,
.destroy_hw = destroy_hw_fd_uobject,
.remove_handle = remove_handle_fd_uobject,
};
EXPORT_SYMBOL(uverbs_fd_class);
struct ib_uobject *
uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
s64 id, struct uverbs_attr_bundle *attrs)
{
const struct uverbs_api_object *obj =
uapi_get_object(attrs->ufile->device->uapi, object_id);
switch (access) {
case UVERBS_ACCESS_READ:
return rdma_lookup_get_uobject(obj, attrs->ufile, id,
UVERBS_LOOKUP_READ, attrs);
case UVERBS_ACCESS_DESTROY:
/* Actual destruction is done inside uverbs_handle_method */
return rdma_lookup_get_uobject(obj, attrs->ufile, id,
UVERBS_LOOKUP_DESTROY, attrs);
case UVERBS_ACCESS_WRITE:
return rdma_lookup_get_uobject(obj, attrs->ufile, id,
UVERBS_LOOKUP_WRITE, attrs);
case UVERBS_ACCESS_NEW:
return rdma_alloc_begin_uobject(obj, attrs);
default:
WARN_ON(true);
return ERR_PTR(-EOPNOTSUPP);
}
}
void uverbs_finalize_object(struct ib_uobject *uobj,
enum uverbs_obj_access access, bool commit,
struct uverbs_attr_bundle *attrs)
{
/*
* refcounts should be handled at the object level and not at the
* uobject level. Refcounts of the objects themselves are done in
* handlers.
*/
switch (access) {
case UVERBS_ACCESS_READ:
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ);
break;
case UVERBS_ACCESS_WRITE:
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
break;
case UVERBS_ACCESS_DESTROY:
if (uobj)
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
break;
case UVERBS_ACCESS_NEW:
if (commit)
rdma_alloc_commit_uobject(uobj, attrs);
else
rdma_alloc_abort_uobject(uobj, attrs);
break;
default:
WARN_ON(true);
}
}

4
sys/ofed/drivers/infiniband/core/ib_sa_query.c
View File

@ -495,7 +495,7 @@ static void free_sm_ah(struct kref *kref)
{
struct ib_sa_sm_ah *sm_ah = container_of(kref, struct ib_sa_sm_ah, ref);
ib_destroy_ah(sm_ah->ah);
ib_destroy_ah(sm_ah->ah, 0);
kfree(sm_ah);
}
@ -535,7 +535,7 @@ static void update_sm_ah(struct work_struct *work)
ah_attr.grh.dgid.global.interface_id = cpu_to_be64(IB_SA_WELL_KNOWN_GUID);
}
new_ah->ah = ib_create_ah(port->agent->qp->pd, &ah_attr);
new_ah->ah = ib_create_ah(port->agent->qp->pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
if (IS_ERR(new_ah->ah)) {
pr_warn("Couldn't create new SM AH\n");
kfree(new_ah);

6
sys/ofed/drivers/infiniband/core/ib_user_mad.c
View File

@ -201,7 +201,7 @@ static void send_handler(struct ib_mad_agent *agent,
struct ib_umad_packet *packet = send_wc->send_buf->context[0];
dequeue_send(file, packet);
ib_destroy_ah(packet->msg->ah);
ib_destroy_ah(packet->msg->ah, RDMA_DESTROY_AH_SLEEPABLE);
ib_free_send_mad(packet->msg);
if (send_wc->status == IB_WC_RESP_TIMEOUT_ERR) {
@ -509,7 +509,7 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
ah_attr.grh.traffic_class = packet->mad.hdr.traffic_class;
}
ah = ib_create_ah(agent->qp->pd, &ah_attr);
ah = ib_create_user_ah(agent->qp->pd, &ah_attr, NULL);
if (IS_ERR(ah)) {
ret = PTR_ERR(ah);
goto err_up;
@ -603,7 +603,7 @@ err_send:
err_msg:
ib_free_send_mad(packet->msg);
err_ah:
ib_destroy_ah(ah);
ib_destroy_ah(ah, RDMA_DESTROY_AH_SLEEPABLE);
err_up:
mutex_unlock(&file->mutex);
err:

4144
sys/ofed/drivers/infiniband/core/ib_uverbs_cmd.c
View File

File diff suppressed because it is too large Load Diff

767
sys/ofed/drivers/infiniband/core/ib_uverbs_ioctl.c Normal file
View File

@ -0,0 +1,767 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <linux/radix-tree.h>
#include <linux/file.h>
#include <linux/overflow.h>
#include <rdma/rdma_user_ioctl.h>
#include <rdma/uverbs_ioctl.h>
#include "rdma_core.h"
#include "uverbs.h"
struct bundle_alloc_head {
struct bundle_alloc_head *next;
uint8_t data[0];
};
struct bundle_priv {
/* Must be first */
struct bundle_alloc_head alloc_head;
struct bundle_alloc_head *allocated_mem;
size_t internal_avail;
size_t internal_used;
struct radix_tree_root *radix;
const struct uverbs_api_ioctl_method *method_elm;
u32 method_key;
struct ib_uverbs_attr __user *user_attrs;
struct ib_uverbs_attr *uattrs;
DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
/*
* Must be last. bundle ends in a flex array which overlaps
* internal_buffer.
*/
struct uverbs_attr_bundle bundle;
u64 internal_buffer[32];
};
/*
* Each method has an absolute minimum amount of memory it needs to allocate,
* precompute that amount and determine if the onstack memory can be used or
* if allocation is need.
*/
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
unsigned int num_attrs)
{
struct bundle_priv *pbundle;
size_t bundle_size =
offsetof(struct bundle_priv, internal_buffer) +
sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
sizeof(*pbundle->uattrs) * num_attrs;
method_elm->use_stack = bundle_size <= sizeof(*pbundle);
method_elm->bundle_size =
ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
/* Do not want order-2 allocations for this. */
WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
}
/**
* uverbs_alloc() - Quickly allocate memory for use with a bundle
* @bundle: The bundle
* @size: Number of bytes to allocate
* @flags: Allocator flags
*
* The bundle allocator is intended for allocations that are connected with
* processing the system call related to the bundle. The allocated memory is
* always freed once the system call completes, and cannot be freed any other
* way.
*
* This tries to use a small pool of pre-allocated memory for performance.
*/
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
gfp_t flags)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
size_t new_used;
void *res;
new_used = size + pbundle->internal_used;
if (new_used < size)
return ERR_PTR(-EOVERFLOW);
if (new_used > pbundle->internal_avail) {
struct bundle_alloc_head *buf;
buf = kvmalloc(struct_size(buf, data, size), flags);
if (!buf)
return ERR_PTR(-ENOMEM);
buf->next = pbundle->allocated_mem;
pbundle->allocated_mem = buf;
return buf->data;
}
res = (u8 *)pbundle->internal_buffer + pbundle->internal_used;
pbundle->internal_used =
ALIGN(new_used, sizeof(*pbundle->internal_buffer));
if (flags & __GFP_ZERO)
memset(res, 0, size);
return res;
}
EXPORT_SYMBOL(_uverbs_alloc);
static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
u16 len)
{
if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data))
return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data + len),
uattr->len - len);
return !memchr_inv((const u8 *)&uattr->data + len,
0, uattr->len - len);
}
static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
const struct uverbs_attr *attr)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
u16 flags;
flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
UVERBS_ATTR_F_VALID_OUTPUT;
if (put_user(flags,
&pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
return -EFAULT;
return 0;
}
static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
const struct uverbs_api_attr *attr_uapi,
struct uverbs_objs_arr_attr *attr,
struct ib_uverbs_attr *uattr,
u32 attr_bkey)
{
const struct uverbs_attr_spec *spec = &attr_uapi->spec;
size_t array_len;
u32 *idr_vals;
int ret = 0;
size_t i;
if (uattr->attr_data.reserved)
return -EINVAL;
if (uattr->len % sizeof(u32))
return -EINVAL;
array_len = uattr->len / sizeof(u32);
if (array_len < spec->u2.objs_arr.min_len ||
array_len > spec->u2.objs_arr.max_len)
return -EINVAL;
attr->uobjects =
uverbs_alloc(&pbundle->bundle,
array_size(array_len, sizeof(*attr->uobjects)));
if (IS_ERR(attr->uobjects))
return PTR_ERR(attr->uobjects);
/*
* Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
* to store idrs array and avoid additional memory allocation. The
* idrs array is offset to the end of the uobjects array so we will be
* able to read idr and replace with a pointer.
*/
idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
if (uattr->len > sizeof(uattr->data)) {
ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
uattr->len);
if (ret)
return -EFAULT;
} else {
memcpy(idr_vals, &uattr->data, uattr->len);
}
for (i = 0; i != array_len; i++) {
attr->uobjects[i] = uverbs_get_uobject_from_file(
spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
idr_vals[i], &pbundle->bundle);
if (IS_ERR(attr->uobjects[i])) {
ret = PTR_ERR(attr->uobjects[i]);
break;
}
}
attr->len = i;
__set_bit(attr_bkey, pbundle->spec_finalize);
return ret;
}
static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
struct uverbs_objs_arr_attr *attr,
bool commit,
struct uverbs_attr_bundle *attrs)
{
const struct uverbs_attr_spec *spec = &attr_uapi->spec;
size_t i;
for (i = 0; i != attr->len; i++)
uverbs_finalize_object(attr->uobjects[i],
spec->u2.objs_arr.access, commit, attrs);
}
static int uverbs_process_attr(struct bundle_priv *pbundle,
const struct uverbs_api_attr *attr_uapi,
struct ib_uverbs_attr *uattr, u32 attr_bkey)
{
const struct uverbs_attr_spec *spec = &attr_uapi->spec;
struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
const struct uverbs_attr_spec *val_spec = spec;
struct uverbs_obj_attr *o_attr;
switch (spec->type) {
case UVERBS_ATTR_TYPE_ENUM_IN:
if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
return -EOPNOTSUPP;
if (uattr->attr_data.enum_data.reserved)
return -EINVAL;
val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
/* Currently we only support PTR_IN based enums */
if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
return -EOPNOTSUPP;
e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
/* fall through */
case UVERBS_ATTR_TYPE_PTR_IN:
/* Ensure that any data provided by userspace beyond the known
* struct is zero. Userspace that knows how to use some future
* longer struct will fail here if used with an old kernel and
* non-zero content, making ABI compat/discovery simpler.
*/
if (uattr->len > val_spec->u.ptr.len &&
val_spec->zero_trailing &&
!uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
return -EOPNOTSUPP;
/* fall through */
case UVERBS_ATTR_TYPE_PTR_OUT:
if (uattr->len < val_spec->u.ptr.min_len ||
(!val_spec->zero_trailing &&
uattr->len > val_spec->u.ptr.len))
return -EINVAL;
if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
uattr->attr_data.reserved)
return -EINVAL;
e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
e->ptr_attr.len = uattr->len;
if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
void *p;
p = uverbs_alloc(&pbundle->bundle, uattr->len);
if (IS_ERR(p))
return PTR_ERR(p);
e->ptr_attr.ptr = p;
if (copy_from_user(p, u64_to_user_ptr(uattr->data),
uattr->len))
return -EFAULT;
} else {
e->ptr_attr.data = uattr->data;
}
break;
case UVERBS_ATTR_TYPE_IDR:
case UVERBS_ATTR_TYPE_FD:
if (uattr->attr_data.reserved)
return -EINVAL;
if (uattr->len != 0)
return -EINVAL;
o_attr = &e->obj_attr;
o_attr->attr_elm = attr_uapi;
/*
* The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
* s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
* here without caring about truncation as we know that the
* IDR implementation today rejects negative IDs
*/
o_attr->uobject = uverbs_get_uobject_from_file(
spec->u.obj.obj_type, spec->u.obj.access,
uattr->data_s64, &pbundle->bundle);
if (IS_ERR(o_attr->uobject))
return PTR_ERR(o_attr->uobject);
__set_bit(attr_bkey, pbundle->uobj_finalize);
if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
unsigned int uattr_idx = uattr - pbundle->uattrs;
s64 id = o_attr->uobject->id;
/* Copy the allocated id to the user-space */
if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
return -EFAULT;
}
break;
case UVERBS_ATTR_TYPE_IDRS_ARRAY:
return uverbs_process_idrs_array(pbundle, attr_uapi,
&e->objs_arr_attr, uattr,
attr_bkey);
default:
return -EOPNOTSUPP;
}
return 0;
}
static void *uapi_get_attr_for_method(struct bundle_priv *pbundle,
u32 attr_key)
{
return radix_tree_lookup(pbundle->radix,
pbundle->method_key | attr_key);
}
static int uverbs_set_attr(struct bundle_priv *pbundle,
struct ib_uverbs_attr *uattr)
{
u32 attr_key = uapi_key_attr(uattr->attr_id);
u32 attr_bkey = uapi_bkey_attr(attr_key);
const struct uverbs_api_attr *attr;
void *slot;
int ret;
slot = uapi_get_attr_for_method(pbundle, attr_key);
if (!slot) {
/*
* Kernel does not support the attribute but user-space says it
* is mandatory
*/
if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
return -EPROTONOSUPPORT;
return 0;
}
attr = slot;
/* Reject duplicate attributes from user-space */
if (test_bit(attr_bkey, pbundle->bundle.attr_present))
return -EINVAL;
ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
if (ret)
return ret;
__set_bit(attr_bkey, pbundle->bundle.attr_present);
return 0;
}
static int ib_uverbs_run_method(struct bundle_priv *pbundle,
unsigned int num_attrs)
{
int (*handler)(struct uverbs_attr_bundle *attrs);
size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
unsigned int i;
int ret;
/* See uverbs_disassociate_api() */
handler = srcu_dereference(
pbundle->method_elm->handler,
&pbundle->bundle.ufile->device->disassociate_srcu);
if (!handler)
return -EIO;
pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
if (IS_ERR(pbundle->uattrs))
return PTR_ERR(pbundle->uattrs);
if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
return -EFAULT;
for (i = 0; i != num_attrs; i++) {
ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
if (unlikely(ret))
return ret;
}
/* User space did not provide all the mandatory attributes */
if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
pbundle->bundle.attr_present,
pbundle->method_elm->key_bitmap_len)))
return -EINVAL;
if (pbundle->method_elm->has_udata)
uverbs_fill_udata(&pbundle->bundle,
&pbundle->bundle.driver_udata,
UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
else
pbundle->bundle.driver_udata = (struct ib_udata){};
if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
struct uverbs_obj_attr *destroy_attr =
&pbundle->bundle.attrs[destroy_bkey].obj_attr;
ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
if (ret)
return ret;
__clear_bit(destroy_bkey, pbundle->uobj_finalize);
ret = handler(&pbundle->bundle);
uobj_put_destroy(destroy_attr->uobject);
} else {
ret = handler(&pbundle->bundle);
}
/*
* Until the drivers are revised to use the bundle directly we have to
* assume that the driver wrote to its UHW_OUT and flag userspace
* appropriately.
*/
if (!ret && pbundle->method_elm->has_udata) {
const struct uverbs_attr *attr =
uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
if (!IS_ERR(attr))
ret = uverbs_set_output(&pbundle->bundle, attr);
}
/*
* EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
* not invoke the method because the request is not supported. No
* other cases should return this code.
*/
if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
return -EINVAL;
return ret;
}
static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
{
unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
struct bundle_alloc_head *memblock;
unsigned int i;
/* fast path for simple uobjects */
i = -1;
while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
i + 1)) < key_bitmap_len) {
struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
uverbs_finalize_object(
attr->obj_attr.uobject,
attr->obj_attr.attr_elm->spec.u.obj.access, commit,
&pbundle->bundle);
}
i = -1;
while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
i + 1)) < key_bitmap_len) {
struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
const struct uverbs_api_attr *attr_uapi;
void *slot;
slot = uapi_get_attr_for_method(
pbundle,
pbundle->method_key | uapi_bkey_to_key_attr(i));
if (WARN_ON(!slot))
continue;
attr_uapi = slot;
if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
commit, &pbundle->bundle);
}
}
for (memblock = pbundle->allocated_mem; memblock;) {
struct bundle_alloc_head *tmp = memblock;
memblock = memblock->next;
kvfree(tmp);
}
}
static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
struct ib_uverbs_ioctl_hdr *hdr,
struct ib_uverbs_attr __user *user_attrs)
{
const struct uverbs_api_ioctl_method *method_elm;
struct uverbs_api *uapi = ufile->device->uapi;
struct bundle_priv *pbundle;
struct bundle_priv onstack;
void *slot;
int ret;
if (unlikely(hdr->driver_id != uapi->driver_id))
return -EINVAL;
slot = radix_tree_lookup(
&uapi->radix,
uapi_key_obj(hdr->object_id) |
uapi_key_ioctl_method(hdr->method_id));
if (unlikely(!slot))
return -EPROTONOSUPPORT;
method_elm = slot;
if (!method_elm->use_stack) {
pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
if (!pbundle)
return -ENOMEM;
pbundle->internal_avail =
method_elm->bundle_size -
offsetof(struct bundle_priv, internal_buffer);
pbundle->alloc_head.next = NULL;
pbundle->allocated_mem = &pbundle->alloc_head;
} else {
pbundle = &onstack;
pbundle->internal_avail = sizeof(pbundle->internal_buffer);
pbundle->allocated_mem = NULL;
}
/* Space for the pbundle->bundle.attrs flex array */
pbundle->method_elm = method_elm;
pbundle->method_key =
uapi_key_obj(hdr->object_id) |
uapi_key_ioctl_method(hdr->method_id);
pbundle->bundle.ufile = ufile;
pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
pbundle->radix = &uapi->radix;
pbundle->user_attrs = user_attrs;
pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
sizeof(*pbundle->bundle.attrs),
sizeof(*pbundle->internal_buffer));
memset(pbundle->bundle.attr_present, 0,
sizeof(pbundle->bundle.attr_present));
memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
bundle_destroy(pbundle, ret == 0);
return ret;
}
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
{
struct ib_uverbs_file *file = filp->private_data;
struct ib_uverbs_ioctl_hdr __user *user_hdr =
(struct ib_uverbs_ioctl_hdr __user *)arg;
struct ib_uverbs_ioctl_hdr hdr;
int srcu_key;
int err;
if (unlikely(cmd != RDMA_VERBS_IOCTL))
return -ENOIOCTLCMD;
err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
if (err)
return -EFAULT;
if (hdr.length > PAGE_SIZE ||
hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
return -EINVAL;
if (hdr.reserved1 || hdr.reserved2)
return -EPROTONOSUPPORT;
srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
return err;
}
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
const struct uverbs_attr *attr;
u64 flags;
attr = uverbs_attr_get(attrs_bundle, idx);
/* Missing attribute means 0 flags */
if (IS_ERR(attr)) {
*to = 0;
return 0;
}
/*
* New userspace code should use 8 bytes to pass flags, but we
* transparently support old userspaces that were using 4 bytes as
* well.
*/
if (attr->ptr_attr.len == 8)
flags = attr->ptr_attr.data;
else if (attr->ptr_attr.len == 4)
flags = *(const u32 *)&attr->ptr_attr.data;
else
return -EINVAL;
if (flags & ~allowed_bits)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags64);
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
u64 flags;
int ret;
ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
if (ret)
return ret;
if (flags > U32_MAX)
return -EINVAL;
*to = flags;
return 0;
}
EXPORT_SYMBOL(uverbs_get_flags32);
/*
* Fill a ib_udata struct (core or uhw) using the given attribute IDs.
* This is primarily used to convert the UVERBS_ATTR_UHW() into the
* ib_udata format used by the drivers.
*/
void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
struct ib_udata *udata, unsigned int attr_in,
unsigned int attr_out)
{
struct bundle_priv *pbundle =
container_of(bundle, struct bundle_priv, bundle);
const struct uverbs_attr *in =
uverbs_attr_get(&pbundle->bundle, attr_in);
const struct uverbs_attr *out =
uverbs_attr_get(&pbundle->bundle, attr_out);
if (!IS_ERR(in)) {
udata->inlen = in->ptr_attr.len;
if (uverbs_attr_ptr_is_inline(in))
udata->inbuf = (void *)
&pbundle->user_attrs[in->ptr_attr.uattr_idx]
.data;
else
udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
} else {
udata->inbuf = NULL;
udata->inlen = 0;
}
if (!IS_ERR(out)) {
udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
udata->outlen = out->ptr_attr.len;
} else {
udata->outbuf = NULL;
udata->outlen = 0;
}
}
int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
size_t min_size;
if (IS_ERR(attr))
return PTR_ERR(attr);
min_size = min_t(size_t, attr->ptr_attr.len, size);
if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
return -EFAULT;
return uverbs_set_output(bundle, attr);
}
EXPORT_SYMBOL(uverbs_copy_to);
/*
* This is only used if the caller has directly used copy_to_use to write the
* data. It signals to user space that the buffer is filled in.
*/
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
return uverbs_set_output(bundle, attr);
}
int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, s64 lower_bound, u64 upper_bound,
s64 *def_val)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr)) {
if ((PTR_ERR(attr) != -ENOENT) || !def_val)
return PTR_ERR(attr);
*to = *def_val;
} else {
*to = attr->ptr_attr.data;
}
if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
return -EINVAL;
return 0;
}
EXPORT_SYMBOL(_uverbs_get_const);
int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
size_t idx, const void *from, size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
if (size < attr->ptr_attr.len) {
if (clear_user(u64_to_user_ptr(attr->ptr_attr.data + size),
attr->ptr_attr.len - size))
return -EFAULT;
}
return uverbs_copy_to(bundle, idx, from, size);
}

1357
sys/ofed/drivers/infiniband/core/ib_uverbs_main.c
View File

File diff suppressed because it is too large Load Diff

359
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types.c Normal file
View File

@ -0,0 +1,359 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <rdma/uverbs_std_types.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/ib_verbs.h>
#include <linux/file.h>
#include "rdma_core.h"
#include "uverbs.h"
static int uverbs_free_ah(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
return ib_destroy_ah_user((struct ib_ah *)uobject->object,
RDMA_DESTROY_AH_SLEEPABLE,
&attrs->driver_udata);
}
static int uverbs_free_flow(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_flow *flow = (struct ib_flow *)uobject->object;
struct ib_uflow_object *uflow =
container_of(uobject, struct ib_uflow_object, uobject);
struct ib_qp *qp = flow->qp;
int ret;
ret = flow->device->destroy_flow(flow);
if (!ret) {
if (qp)
atomic_dec(&qp->usecnt);
ib_uverbs_flow_resources_free(uflow->resources);
}
return ret;
}
static int uverbs_free_mw(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
return uverbs_dealloc_mw((struct ib_mw *)uobject->object);
}
static int uverbs_free_qp(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_qp *qp = uobject->object;
struct ib_uqp_object *uqp =
container_of(uobject, struct ib_uqp_object, uevent.uobject);
int ret;
/*
* If this is a user triggered destroy then do not allow destruction
* until the user cleans up all the mcast bindings. Unlike in other
* places we forcibly clean up the mcast attachments for !DESTROY
* because the mcast attaches are not ubojects and will not be
* destroyed by anything else during cleanup processing.
*/
if (why == RDMA_REMOVE_DESTROY) {
if (!list_empty(&uqp->mcast_list))
return -EBUSY;
} else if (qp == qp->real_qp) {
ib_uverbs_detach_umcast(qp, uqp);
}
ret = ib_destroy_qp_user(qp, &attrs->driver_udata);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
if (uqp->uxrcd)
atomic_dec(&uqp->uxrcd->refcnt);
ib_uverbs_release_uevent(&uqp->uevent);
return ret;
}
static int uverbs_free_rwq_ind_tbl(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_rwq_ind_table *rwq_ind_tbl = uobject->object;
struct ib_wq **ind_tbl = rwq_ind_tbl->ind_tbl;
int ret;
ret = ib_destroy_rwq_ind_table(rwq_ind_tbl);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
kfree(ind_tbl);
return ret;
}
static int uverbs_free_wq(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_wq *wq = uobject->object;
struct ib_uwq_object *uwq =
container_of(uobject, struct ib_uwq_object, uevent.uobject);
int ret;
ret = ib_destroy_wq(wq, &attrs->driver_udata);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
ib_uverbs_release_uevent(&uwq->uevent);
return ret;
}
static int uverbs_free_srq(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_srq *srq = uobject->object;
struct ib_uevent_object *uevent =
container_of(uobject, struct ib_uevent_object, uobject);
enum ib_srq_type srq_type = srq->srq_type;
int ret;
ret = ib_destroy_srq_user(srq, &attrs->driver_udata);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
if (srq_type == IB_SRQT_XRC) {
struct ib_usrq_object *us =
container_of(uevent, struct ib_usrq_object, uevent);
atomic_dec(&us->uxrcd->refcnt);
}
ib_uverbs_release_uevent(uevent);
return ret;
}
static int uverbs_free_xrcd(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_xrcd *xrcd = uobject->object;
struct ib_uxrcd_object *uxrcd =
container_of(uobject, struct ib_uxrcd_object, uobject);
int ret;
ret = ib_destroy_usecnt(&uxrcd->refcnt, why, uobject);
if (ret)
return ret;
mutex_lock(&attrs->ufile->device->xrcd_tree_mutex);
ret = ib_uverbs_dealloc_xrcd(uobject, xrcd, why, attrs);
mutex_unlock(&attrs->ufile->device->xrcd_tree_mutex);
return ret;
}
static int uverbs_free_pd(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_pd *pd = uobject->object;
int ret;
ret = ib_destroy_usecnt(&pd->usecnt, why, uobject);
if (ret)
return ret;
ib_dealloc_pd_user(pd, &attrs->driver_udata);
return 0;
}
void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue)
{
struct ib_uverbs_event *entry, *tmp;
spin_lock_irq(&event_queue->lock);
/*
* The user must ensure that no new items are added to the event_list
* once is_closed is set.
*/
event_queue->is_closed = 1;
spin_unlock_irq(&event_queue->lock);
wake_up_interruptible(&event_queue->poll_wait);
kill_fasync(&event_queue->async_queue, SIGIO, POLL_IN);
spin_lock_irq(&event_queue->lock);
list_for_each_entry_safe(entry, tmp, &event_queue->event_list, list) {
if (entry->counter)
list_del(&entry->obj_list);
list_del(&entry->list);
kfree(entry);
}
spin_unlock_irq(&event_queue->lock);
}
static int
uverbs_completion_event_file_destroy_uobj(struct ib_uobject *uobj,
enum rdma_remove_reason why)
{
struct ib_uverbs_completion_event_file *file =
container_of(uobj, struct ib_uverbs_completion_event_file,
uobj);
ib_uverbs_free_event_queue(&file->ev_queue);
return 0;
}
int uverbs_destroy_def_handler(struct uverbs_attr_bundle *attrs)
{
return 0;
}
EXPORT_SYMBOL(uverbs_destroy_def_handler);
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_completion_event_file),
uverbs_completion_event_file_destroy_uobj,
&uverbs_event_fops,
"[infinibandevent]",
FMODE_READ));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_QP,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uqp_object), uverbs_free_qp));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_MW_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MW_HANDLE,
UVERBS_OBJECT_MW,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_MW,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_mw),
&UVERBS_METHOD(UVERBS_METHOD_MW_DESTROY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_SRQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_usrq_object),
uverbs_free_srq));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_AH_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_AH_HANDLE,
UVERBS_OBJECT_AH,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_AH,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_ah),
&UVERBS_METHOD(UVERBS_METHOD_AH_DESTROY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_FLOW_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_FLOW_HANDLE,
UVERBS_OBJECT_FLOW,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_FLOW,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uflow_object),
uverbs_free_flow),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_DESTROY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_WQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uwq_object), uverbs_free_wq));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_RWQ_IND_TBL_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_RWQ_IND_TBL_HANDLE,
UVERBS_OBJECT_RWQ_IND_TBL,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_RWQ_IND_TBL,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_rwq_ind_tbl),
&UVERBS_METHOD(UVERBS_METHOD_RWQ_IND_TBL_DESTROY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_XRCD_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_XRCD_HANDLE,
UVERBS_OBJECT_XRCD,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_XRCD,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uxrcd_object),
uverbs_free_xrcd),
&UVERBS_METHOD(UVERBS_METHOD_XRCD_DESTROY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_PD_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_PD_HANDLE,
UVERBS_OBJECT_PD,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_PD,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_pd),
&UVERBS_METHOD(UVERBS_METHOD_PD_DESTROY));
const struct uapi_definition uverbs_def_obj_intf[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_PD,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COMP_CHANNEL,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_pd)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_QP,
UAPI_DEF_OBJ_NEEDS_FN(destroy_qp)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_AH,
UAPI_DEF_OBJ_NEEDS_FN(destroy_ah)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MW,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_mw)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_SRQ,
UAPI_DEF_OBJ_NEEDS_FN(destroy_srq)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_FLOW,
UAPI_DEF_OBJ_NEEDS_FN(destroy_flow)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_WQ,
UAPI_DEF_OBJ_NEEDS_FN(destroy_wq)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
UVERBS_OBJECT_RWQ_IND_TBL,
UAPI_DEF_OBJ_NEEDS_FN(destroy_rwq_ind_table)),
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_XRCD,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_xrcd)),
{}
};

54
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_async_fd.c Normal file
View File

@ -0,0 +1,54 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2019, Mellanox Technologies inc. All rights reserved.
*/
#include <rdma/uverbs_std_types.h>
#include <rdma/uverbs_ioctl.h>
#include "rdma_core.h"
#include "uverbs.h"
#include <sys/fcntl.h>
static int UVERBS_HANDLER(UVERBS_METHOD_ASYNC_EVENT_ALLOC)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_METHOD_ASYNC_EVENT_ALLOC);
ib_uverbs_init_async_event_file(
container_of(uobj, struct ib_uverbs_async_event_file, uobj));
return 0;
}
static int uverbs_async_event_destroy_uobj(struct ib_uobject *uobj,
enum rdma_remove_reason why)
{
struct ib_uverbs_async_event_file *event_file =
container_of(uobj, struct ib_uverbs_async_event_file, uobj);
ib_unregister_event_handler(&event_file->event_handler);
ib_uverbs_free_event_queue(&event_file->ev_queue);
return 0;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_ASYNC_EVENT_ALLOC,
UVERBS_ATTR_FD(UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE,
UVERBS_OBJECT_ASYNC_EVENT,
UVERBS_ACCESS_NEW,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_ASYNC_EVENT,
UVERBS_TYPE_ALLOC_FD(sizeof(struct ib_uverbs_async_event_file),
uverbs_async_event_destroy_uobj,
&uverbs_async_event_fops,
"[infinibandevent]",
FMODE_READ),
&UVERBS_METHOD(UVERBS_METHOD_ASYNC_EVENT_ALLOC));
const struct uapi_definition uverbs_def_obj_async_fd[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_ASYNC_EVENT),
{}
};

161
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_counters.c Normal file
View File

@ -0,0 +1,161 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
#include <linux/overflow.h>
static int uverbs_free_counters(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_counters *counters = uobject->object;
int ret;
ret = ib_destroy_usecnt(&counters->usecnt, why, uobject);
if (ret)
return ret;
return counters->device->destroy_counters(counters);
}
static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_CREATE)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_CREATE_COUNTERS_HANDLE);
struct ib_device *ib_dev = attrs->context->device;
struct ib_counters *counters;
int ret;
/*
* This check should be removed once the infrastructure
* have the ability to remove methods from parse tree once
* such condition is met.
*/
if (!ib_dev->create_counters)
return -EOPNOTSUPP;
counters = ib_dev->create_counters(ib_dev, attrs);
if (IS_ERR(counters)) {
ret = PTR_ERR(counters);
goto err_create_counters;
}
counters->device = ib_dev;
counters->uobject = uobj;
uobj->object = counters;
atomic_set(&counters->usecnt, 0);
return 0;
err_create_counters:
return ret;
}
static int UVERBS_HANDLER(UVERBS_METHOD_COUNTERS_READ)(
struct uverbs_attr_bundle *attrs)
{
struct ib_counters_read_attr read_attr = {};
const struct uverbs_attr *uattr;
struct ib_counters *counters =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_READ_COUNTERS_HANDLE);
int ret;
if (!counters->device->read_counters)
return -EOPNOTSUPP;
if (!atomic_read(&counters->usecnt))
return -EINVAL;
ret = uverbs_get_flags32(&read_attr.flags, attrs,
UVERBS_ATTR_READ_COUNTERS_FLAGS,
IB_UVERBS_READ_COUNTERS_PREFER_CACHED);
if (ret)
return ret;
uattr = uverbs_attr_get(attrs, UVERBS_ATTR_READ_COUNTERS_BUFF);
read_attr.ncounters = uattr->ptr_attr.len / sizeof(u64);
read_attr.counters_buff = uverbs_zalloc(
attrs, array_size(read_attr.ncounters, sizeof(u64)));
if (IS_ERR(read_attr.counters_buff))
return PTR_ERR(read_attr.counters_buff);
ret = counters->device->read_counters(counters, &read_attr, attrs);
if (ret)
return ret;
return uverbs_copy_to(attrs, UVERBS_ATTR_READ_COUNTERS_BUFF,
read_attr.counters_buff,
read_attr.ncounters * sizeof(u64));
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_COUNTERS_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_NEW,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_COUNTERS_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_COUNTERS_READ,
UVERBS_ATTR_IDR(UVERBS_ATTR_READ_COUNTERS_HANDLE,
UVERBS_OBJECT_COUNTERS,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_READ_COUNTERS_BUFF,
UVERBS_ATTR_MIN_SIZE(0),
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_READ_COUNTERS_FLAGS,
enum ib_uverbs_read_counters_flags));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_COUNTERS,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_counters),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_COUNTERS_READ));
const struct uapi_definition uverbs_def_obj_counters[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_COUNTERS,
UAPI_DEF_OBJ_NEEDS_FN(destroy_counters)),
{}
};

214
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_cq.c Normal file
View File

@ -0,0 +1,214 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include <rdma/uverbs_std_types.h>
#include "rdma_core.h"
#include "uverbs.h"
static int uverbs_free_cq(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_cq *cq = uobject->object;
struct ib_uverbs_event_queue *ev_queue = cq->cq_context;
struct ib_ucq_object *ucq =
container_of(uobject, struct ib_ucq_object, uevent.uobject);
int ret;
ret = ib_destroy_cq_user(cq, &attrs->driver_udata);
if (ib_is_destroy_retryable(ret, why, uobject))
return ret;
ib_uverbs_release_ucq(
ev_queue ? container_of(ev_queue,
struct ib_uverbs_completion_event_file,
ev_queue) :
NULL,
ucq);
return ret;
}
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_CREATE)(
struct uverbs_attr_bundle *attrs)
{
struct ib_ucq_object *obj = container_of(
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_HANDLE),
typeof(*obj), uevent.uobject);
struct ib_device *ib_dev = attrs->context->device;
int ret;
u64 user_handle;
struct ib_cq_init_attr attr = {};
struct ib_cq *cq;
struct ib_uverbs_completion_event_file *ev_file = NULL;
struct ib_uobject *ev_file_uobj;
if (!ib_dev->create_cq || !ib_dev->destroy_cq)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.comp_vector, attrs,
UVERBS_ATTR_CREATE_CQ_COMP_VECTOR);
if (!ret)
ret = uverbs_copy_from(&attr.cqe, attrs,
UVERBS_ATTR_CREATE_CQ_CQE);
if (!ret)
ret = uverbs_copy_from(&user_handle, attrs,
UVERBS_ATTR_CREATE_CQ_USER_HANDLE);
if (ret)
return ret;
ret = uverbs_get_flags32(&attr.flags, attrs,
UVERBS_ATTR_CREATE_CQ_FLAGS,
IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION |
IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN);
if (ret)
return ret;
ev_file_uobj = uverbs_attr_get_uobject(attrs, UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL);
if (!IS_ERR(ev_file_uobj)) {
ev_file = container_of(ev_file_uobj,
struct ib_uverbs_completion_event_file,
uobj);
uverbs_uobject_get(ev_file_uobj);
}
if (attr.comp_vector >= attrs->ufile->device->num_comp_vectors) {
ret = -EINVAL;
goto err_event_file;
}
INIT_LIST_HEAD(&obj->comp_list);
INIT_LIST_HEAD(&obj->uevent.event_list);
cq = rdma_zalloc_drv_obj(ib_dev, ib_cq);
if (!cq) {
ret = -ENOMEM;
goto err_event_file;
}
cq->device = ib_dev;
cq->uobject = obj;
cq->comp_handler = ib_uverbs_comp_handler;
cq->event_handler = ib_uverbs_cq_event_handler;
cq->cq_context = ev_file ? &ev_file->ev_queue : NULL;
atomic_set(&cq->usecnt, 0);
ret = ib_dev->create_cq(cq, &attr, &attrs->driver_udata);
if (ret)
goto err_free;
obj->uevent.uobject.object = cq;
obj->uevent.uobject.user_handle = user_handle;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_CREATE_CQ_RESP_CQE, &cq->cqe,
sizeof(cq->cqe));
if (ret)
goto err_cq;
return 0;
err_cq:
ib_destroy_cq_user(cq, uverbs_get_cleared_udata(attrs));
cq = NULL;
err_free:
kfree(cq);
err_event_file:
if (ev_file)
uverbs_uobject_put(ev_file_uobj);
return ret;
};
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_CQ_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_CQ_HANDLE,
UVERBS_OBJECT_CQ,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_CQE,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_USER_HANDLE,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_FD(UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL,
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_ACCESS_READ,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CREATE_CQ_COMP_VECTOR,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_CREATE_CQ_FLAGS,
enum ib_uverbs_ex_create_cq_flags),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CREATE_CQ_RESP_CQE,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_UHW());
static int UVERBS_HANDLER(UVERBS_METHOD_CQ_DESTROY)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_DESTROY_CQ_HANDLE);
struct ib_ucq_object *obj =
container_of(uobj, struct ib_ucq_object, uevent.uobject);
struct ib_uverbs_destroy_cq_resp resp = {
.comp_events_reported = obj->comp_events_reported,
.async_events_reported = obj->uevent.events_reported
};
return uverbs_copy_to(attrs, UVERBS_ATTR_DESTROY_CQ_RESP, &resp,
sizeof(resp));
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_CQ_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_CQ_HANDLE,
UVERBS_OBJECT_CQ,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_DESTROY_CQ_RESP,
UVERBS_ATTR_TYPE(struct ib_uverbs_destroy_cq_resp),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_CQ,
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_ucq_object), uverbs_free_cq),
#if 1 /* CONFIG_INFINIBAND_EXP_LEGACY_VERBS_NEW_UAPI */
&UVERBS_METHOD(UVERBS_METHOD_CQ_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_CQ_DESTROY)
#endif
);
const struct uapi_definition uverbs_def_obj_cq[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_CQ,
UAPI_DEF_OBJ_NEEDS_FN(destroy_cq)),
{}
};

261
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_device.c Normal file
View File

@ -0,0 +1,261 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*/
#include <rdma/uverbs_std_types.h>
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_ioctl.h>
/*
* This ioctl method allows calling any defined write or write_ex
* handler. This essentially replaces the hdr/ex_hdr system with the ioctl
* marshalling, and brings the non-ex path into the same marshalling as the ex
* path.
*/
static int UVERBS_HANDLER(UVERBS_METHOD_INVOKE_WRITE)(
struct uverbs_attr_bundle *attrs)
{
struct uverbs_api *uapi = attrs->ufile->device->uapi;
const struct uverbs_api_write_method *method_elm;
u32 cmd;
int rc;
rc = uverbs_get_const(&cmd, attrs, UVERBS_ATTR_WRITE_CMD);
if (rc)
return rc;
method_elm = uapi_get_method(uapi, cmd);
if (IS_ERR(method_elm))
return PTR_ERR(method_elm);
uverbs_fill_udata(attrs, &attrs->ucore, UVERBS_ATTR_CORE_IN,
UVERBS_ATTR_CORE_OUT);
if (attrs->ucore.inlen < method_elm->req_size ||
attrs->ucore.outlen < method_elm->resp_size)
return -ENOSPC;
return method_elm->handler(attrs);
}
DECLARE_UVERBS_NAMED_METHOD(UVERBS_METHOD_INVOKE_WRITE,
UVERBS_ATTR_CONST_IN(UVERBS_ATTR_WRITE_CMD,
enum ib_uverbs_write_cmds,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_CORE_IN,
UVERBS_ATTR_MIN_SIZE(sizeof(u32)),
UA_OPTIONAL),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_CORE_OUT,
UVERBS_ATTR_MIN_SIZE(0),
UA_OPTIONAL),
UVERBS_ATTR_UHW());
static uint32_t *
gather_objects_handle(struct ib_uverbs_file *ufile,
const struct uverbs_api_object *uapi_object,
struct uverbs_attr_bundle *attrs,
ssize_t out_len,
u64 *total)
{
u64 max_count = out_len / sizeof(u32);
struct ib_uobject *obj;
u64 count = 0;
u32 *handles;
/* Allocated memory that cannot page out where we gather
* all object ids under a spin_lock.
*/
handles = uverbs_zalloc(attrs, out_len);
if (IS_ERR(handles))
return handles;
spin_lock_irq(&ufile->uobjects_lock);
list_for_each_entry(obj, &ufile->uobjects, list) {
u32 obj_id = obj->id;
if (obj->uapi_object != uapi_object)
continue;
if (count >= max_count)
break;
handles[count] = obj_id;
count++;
}
spin_unlock_irq(&ufile->uobjects_lock);
*total = count;
return handles;
}
static int UVERBS_HANDLER(UVERBS_METHOD_INFO_HANDLES)(
struct uverbs_attr_bundle *attrs)
{
const struct uverbs_api_object *uapi_object;
ssize_t out_len;
u64 total = 0;
u16 object_id;
u32 *handles;
int ret;
out_len = uverbs_attr_get_len(attrs, UVERBS_ATTR_INFO_HANDLES_LIST);
if (out_len <= 0 || (out_len % sizeof(u32) != 0))
return -EINVAL;
ret = uverbs_get_const(&object_id, attrs, UVERBS_ATTR_INFO_OBJECT_ID);
if (ret)
return ret;
uapi_object = uapi_get_object(attrs->ufile->device->uapi, object_id);
if (!uapi_object)
return -EINVAL;
handles = gather_objects_handle(attrs->ufile, uapi_object, attrs,
out_len, &total);
if (IS_ERR(handles))
return PTR_ERR(handles);
ret = uverbs_copy_to(attrs, UVERBS_ATTR_INFO_HANDLES_LIST, handles,
sizeof(u32) * total);
if (ret)
goto err;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_INFO_TOTAL_HANDLES, &total,
sizeof(total));
err:
return ret;
}
void copy_port_attr_to_resp(struct ib_port_attr *attr,
struct ib_uverbs_query_port_resp *resp,
struct ib_device *ib_dev, u8 port_num)
{
resp->state = attr->state;
resp->max_mtu = attr->max_mtu;
resp->active_mtu = attr->active_mtu;
resp->gid_tbl_len = attr->gid_tbl_len;
resp->port_cap_flags = make_port_cap_flags(attr);
resp->max_msg_sz = attr->max_msg_sz;
resp->bad_pkey_cntr = attr->bad_pkey_cntr;
resp->qkey_viol_cntr = attr->qkey_viol_cntr;
resp->pkey_tbl_len = attr->pkey_tbl_len;
if (attr->grh_required)
resp->flags |= IB_UVERBS_QPF_GRH_REQUIRED;
resp->lid = (u16)attr->lid;
resp->sm_lid = (u16)attr->sm_lid;
resp->lmc = attr->lmc;
resp->max_vl_num = attr->max_vl_num;
resp->sm_sl = attr->sm_sl;
resp->subnet_timeout = attr->subnet_timeout;
resp->init_type_reply = attr->init_type_reply;
resp->active_width = attr->active_width;
resp->active_speed = attr->active_speed;
resp->phys_state = attr->phys_state;
resp->link_layer = rdma_port_get_link_layer(ib_dev, port_num);
}
static int UVERBS_HANDLER(UVERBS_METHOD_QUERY_PORT)(
struct uverbs_attr_bundle *attrs)
{
struct ib_device *ib_dev;
struct ib_port_attr attr = {};
struct ib_uverbs_query_port_resp_ex resp = {};
struct ib_ucontext *ucontext;
int ret;
u8 port_num;
ucontext = ib_uverbs_get_ucontext(attrs);
if (IS_ERR(ucontext))
return PTR_ERR(ucontext);
ib_dev = ucontext->device;
/* FIXME: Extend the UAPI_DEF_OBJ_NEEDS_FN stuff.. */
if (!ib_dev->query_port)
return -EOPNOTSUPP;
ret = uverbs_get_const(&port_num, attrs,
UVERBS_ATTR_QUERY_PORT_PORT_NUM);
if (ret)
return ret;
ret = ib_query_port(ib_dev, port_num, &attr);
if (ret)
return ret;
copy_port_attr_to_resp(&attr, &resp.legacy_resp, ib_dev, port_num);
resp.port_cap_flags2 = 0;
return uverbs_copy_to_struct_or_zero(attrs, UVERBS_ATTR_QUERY_PORT_RESP,
&resp, sizeof(resp));
}
static int UVERBS_HANDLER(UVERBS_METHOD_GET_CONTEXT)(
struct uverbs_attr_bundle *attrs)
{
u32 num_comp = attrs->ufile->device->num_comp_vectors;
u64 core_support = IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS;
int ret;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
&num_comp, sizeof(num_comp));
if (IS_UVERBS_COPY_ERR(ret))
return ret;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,
&core_support, sizeof(core_support));
if (IS_UVERBS_COPY_ERR(ret))
return ret;
ret = ib_alloc_ucontext(attrs);
if (ret)
return ret;
ret = ib_init_ucontext(attrs);
if (ret) {
kfree(attrs->context);
attrs->context = NULL;
return ret;
}
return 0;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_GET_CONTEXT,
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,
UVERBS_ATTR_TYPE(u64), UA_OPTIONAL),
UVERBS_ATTR_UHW());
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_INFO_HANDLES,
/* Also includes any device specific object ids */
UVERBS_ATTR_CONST_IN(UVERBS_ATTR_INFO_OBJECT_ID,
enum uverbs_default_objects, UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_INFO_TOTAL_HANDLES,
UVERBS_ATTR_TYPE(u32), UA_OPTIONAL),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_INFO_HANDLES_LIST,
UVERBS_ATTR_MIN_SIZE(sizeof(u32)), UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_QUERY_PORT,
UVERBS_ATTR_CONST_IN(UVERBS_ATTR_QUERY_PORT_PORT_NUM, u8, UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(
UVERBS_ATTR_QUERY_PORT_RESP,
UVERBS_ATTR_STRUCT(struct ib_uverbs_query_port_resp_ex,
reserved),
UA_MANDATORY));
DECLARE_UVERBS_GLOBAL_METHODS(UVERBS_OBJECT_DEVICE,
&UVERBS_METHOD(UVERBS_METHOD_GET_CONTEXT),
&UVERBS_METHOD(UVERBS_METHOD_INVOKE_WRITE),
&UVERBS_METHOD(UVERBS_METHOD_INFO_HANDLES),
&UVERBS_METHOD(UVERBS_METHOD_QUERY_PORT));
const struct uapi_definition uverbs_def_obj_device[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DEVICE),
{},
};

118
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_dm.c Normal file
View File

@ -0,0 +1,118 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_dm(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_dm *dm = uobject->object;
int ret;
ret = ib_destroy_usecnt(&dm->usecnt, why, uobject);
if (ret)
return ret;
return dm->device->dealloc_dm(dm, attrs);
}
static int UVERBS_HANDLER(UVERBS_METHOD_DM_ALLOC)(
struct uverbs_attr_bundle *attrs)
{
struct ib_dm_alloc_attr attr = {};
struct ib_uobject *uobj =
uverbs_attr_get(attrs, UVERBS_ATTR_ALLOC_DM_HANDLE)
->obj_attr.uobject;
struct ib_device *ib_dev = attrs->context->device;
struct ib_dm *dm;
int ret;
if (!ib_dev->alloc_dm)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.length, attrs,
UVERBS_ATTR_ALLOC_DM_LENGTH);
if (ret)
return ret;
ret = uverbs_copy_from(&attr.alignment, attrs,
UVERBS_ATTR_ALLOC_DM_ALIGNMENT);
if (ret)
return ret;
dm = ib_dev->alloc_dm(ib_dev, attrs->context, &attr, attrs);
if (IS_ERR(dm))
return PTR_ERR(dm);
dm->device = ib_dev;
dm->length = attr.length;
dm->uobject = uobj;
atomic_set(&dm->usecnt, 0);
uobj->object = dm;
return 0;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DM_ALLOC,
UVERBS_ATTR_IDR(UVERBS_ATTR_ALLOC_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DM_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ALLOC_DM_ALIGNMENT,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_DM_FREE,
UVERBS_ATTR_IDR(UVERBS_ATTR_FREE_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(UVERBS_OBJECT_DM,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_dm),
&UVERBS_METHOD(UVERBS_METHOD_DM_ALLOC),
&UVERBS_METHOD(UVERBS_METHOD_DM_FREE));
const struct uapi_definition uverbs_def_obj_dm[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_DM,
UAPI_DEF_OBJ_NEEDS_FN(dealloc_dm)),
{}
};

449
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_flow_action.c Normal file
View File

@ -0,0 +1,449 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_flow_action(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
struct ib_flow_action *action = uobject->object;
int ret;
ret = ib_destroy_usecnt(&action->usecnt, why, uobject);
if (ret)
return ret;
return action->device->destroy_flow_action(action);
}
static u64 esp_flags_uverbs_to_verbs(struct uverbs_attr_bundle *attrs,
u32 flags, bool is_modify)
{
u64 verbs_flags = flags;
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ESN))
verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_ESN_TRIGGERED;
if (is_modify && uverbs_attr_is_valid(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS))
verbs_flags |= IB_FLOW_ACTION_ESP_FLAGS_MOD_ESP_ATTRS;
return verbs_flags;
};
static int validate_flow_action_esp_keymat_aes_gcm(struct ib_flow_action_attrs_esp_keymats *keymat)
{
struct ib_uverbs_flow_action_esp_keymat_aes_gcm *aes_gcm =
&keymat->keymat.aes_gcm;
if (aes_gcm->iv_algo > IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ)
return -EOPNOTSUPP;
if (aes_gcm->key_len != 32 &&
aes_gcm->key_len != 24 &&
aes_gcm->key_len != 16)
return -EINVAL;
if (aes_gcm->icv_len != 16 &&
aes_gcm->icv_len != 8 &&
aes_gcm->icv_len != 12)
return -EINVAL;
return 0;
}
static int (* const flow_action_esp_keymat_validate[])(struct ib_flow_action_attrs_esp_keymats *keymat) = {
[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = validate_flow_action_esp_keymat_aes_gcm,
};
static int flow_action_esp_replay_none(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify)
{
/* This is used in order to modify an esp flow action with an enabled
* replay protection to a disabled one. This is only supported via
* modify, as in create verb we can simply drop the REPLAY attribute and
* achieve the same thing.
*/
return is_modify ? 0 : -EINVAL;
}
static int flow_action_esp_replay_def_ok(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify)
{
/* Some replay protections could always be enabled without validating
* anything.
*/
return 0;
}
static int (* const flow_action_esp_replay_validate[])(struct ib_flow_action_attrs_esp_replays *replay,
bool is_modify) = {
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = flow_action_esp_replay_none,
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = flow_action_esp_replay_def_ok,
};
static int parse_esp_ip(enum ib_flow_spec_type proto,
const void __user *val_ptr,
size_t len, union ib_flow_spec *out)
{
int ret;
const struct ib_uverbs_flow_ipv4_filter ipv4 = {
.src_ip = cpu_to_be32(0xffffffffUL),
.dst_ip = cpu_to_be32(0xffffffffUL),
.proto = 0xff,
.tos = 0xff,
.ttl = 0xff,
.flags = 0xff,
};
const struct ib_uverbs_flow_ipv6_filter ipv6 = {
.src_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
.dst_ip = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff},
.flow_label = cpu_to_be32(0xffffffffUL),
.next_hdr = 0xff,
.traffic_class = 0xff,
.hop_limit = 0xff,
};
union {
struct ib_uverbs_flow_ipv4_filter ipv4;
struct ib_uverbs_flow_ipv6_filter ipv6;
} user_val = {};
const void *user_pmask;
size_t val_len;
/* If the flow IPv4/IPv6 flow specifications are extended, the mask
* should be changed as well.
*/
BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv4_filter, flags) +
sizeof(ipv4.flags) != sizeof(ipv4));
BUILD_BUG_ON(offsetof(struct ib_uverbs_flow_ipv6_filter, reserved) +
sizeof(ipv6.reserved) != sizeof(ipv6));
switch (proto) {
case IB_FLOW_SPEC_IPV4:
if (len > sizeof(user_val.ipv4) &&
!ib_is_buffer_cleared((const u8 *)val_ptr + sizeof(user_val.ipv4),
len - sizeof(user_val.ipv4)))
return -EOPNOTSUPP;
val_len = min_t(size_t, len, sizeof(user_val.ipv4));
ret = copy_from_user(&user_val.ipv4, val_ptr,
val_len);
if (ret)
return -EFAULT;
user_pmask = &ipv4;
break;
case IB_FLOW_SPEC_IPV6:
if (len > sizeof(user_val.ipv6) &&
!ib_is_buffer_cleared((const u8 *)val_ptr + sizeof(user_val.ipv6),
len - sizeof(user_val.ipv6)))
return -EOPNOTSUPP;
val_len = min_t(size_t, len, sizeof(user_val.ipv6));
ret = copy_from_user(&user_val.ipv6, val_ptr,
val_len);
if (ret)
return -EFAULT;
user_pmask = &ipv6;
break;
default:
return -EOPNOTSUPP;
}
return ib_uverbs_kern_spec_to_ib_spec_filter(proto, user_pmask,
&user_val,
val_len, out);
}
static int flow_action_esp_get_encap(struct ib_flow_spec_list *out,
struct uverbs_attr_bundle *attrs)
{
struct ib_uverbs_flow_action_esp_encap uverbs_encap;
int ret;
ret = uverbs_copy_from(&uverbs_encap, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP);
if (ret)
return ret;
/* We currently support only one encap */
if (uverbs_encap.next_ptr)
return -EOPNOTSUPP;
if (uverbs_encap.type != IB_FLOW_SPEC_IPV4 &&
uverbs_encap.type != IB_FLOW_SPEC_IPV6)
return -EOPNOTSUPP;
return parse_esp_ip(uverbs_encap.type,
u64_to_user_ptr(uverbs_encap.val_ptr),
uverbs_encap.len,
&out->spec);
}
struct ib_flow_action_esp_attr {
struct ib_flow_action_attrs_esp hdr;
struct ib_flow_action_attrs_esp_keymats keymat;
struct ib_flow_action_attrs_esp_replays replay;
/* We currently support only one spec */
struct ib_flow_spec_list encap;
};
#define ESP_LAST_SUPPORTED_FLAG IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW
static int parse_flow_action_esp(struct ib_device *ib_dev,
struct uverbs_attr_bundle *attrs,
struct ib_flow_action_esp_attr *esp_attr,
bool is_modify)
{
struct ib_uverbs_flow_action_esp uverbs_esp = {};
int ret;
/* Optional param, if it doesn't exist, we get -ENOENT and skip it */
ret = uverbs_copy_from(&esp_attr->hdr.esn, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ESN);
if (IS_UVERBS_COPY_ERR(ret))
return ret;
/* This can be called from FLOW_ACTION_ESP_MODIFY where
* UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS is optional
*/
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS)) {
ret = uverbs_copy_from_or_zero(&uverbs_esp, attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS);
if (ret)
return ret;
if (uverbs_esp.flags & ~((ESP_LAST_SUPPORTED_FLAG << 1) - 1))
return -EOPNOTSUPP;
esp_attr->hdr.spi = uverbs_esp.spi;
esp_attr->hdr.seq = uverbs_esp.seq;
esp_attr->hdr.tfc_pad = uverbs_esp.tfc_pad;
esp_attr->hdr.hard_limit_pkts = uverbs_esp.hard_limit_pkts;
}
esp_attr->hdr.flags = esp_flags_uverbs_to_verbs(attrs, uverbs_esp.flags,
is_modify);
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT)) {
esp_attr->keymat.protocol =
uverbs_attr_get_enum_id(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);
ret = uverbs_copy_from_or_zero(&esp_attr->keymat.keymat,
attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT);
if (ret)
return ret;
ret = flow_action_esp_keymat_validate[esp_attr->keymat.protocol](&esp_attr->keymat);
if (ret)
return ret;
esp_attr->hdr.keymat = &esp_attr->keymat;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY)) {
esp_attr->replay.protocol =
uverbs_attr_get_enum_id(attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);
ret = uverbs_copy_from_or_zero(&esp_attr->replay.replay,
attrs,
UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY);
if (ret)
return ret;
ret = flow_action_esp_replay_validate[esp_attr->replay.protocol](&esp_attr->replay,
is_modify);
if (ret)
return ret;
esp_attr->hdr.replay = &esp_attr->replay;
}
if (uverbs_attr_is_valid(attrs, UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP)) {
ret = flow_action_esp_get_encap(&esp_attr->encap, attrs);
if (ret)
return ret;
esp_attr->hdr.encap = &esp_attr->encap;
}
return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE);
struct ib_device *ib_dev = attrs->context->device;
int ret;
struct ib_flow_action *action;
struct ib_flow_action_esp_attr esp_attr = {};
if (!ib_dev->create_flow_action_esp)
return -EOPNOTSUPP;
ret = parse_flow_action_esp(ib_dev, attrs, &esp_attr, false);
if (ret)
return ret;
/* No need to check as this attribute is marked as MANDATORY */
action = ib_dev->create_flow_action_esp(ib_dev, &esp_attr.hdr,
attrs);
if (IS_ERR(action))
return PTR_ERR(action);
uverbs_flow_action_fill_action(action, uobj, ib_dev,
IB_FLOW_ACTION_ESP);
return 0;
}
static int UVERBS_HANDLER(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY)(
struct uverbs_attr_bundle *attrs)
{
struct ib_uobject *uobj = uverbs_attr_get_uobject(
attrs, UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE);
struct ib_flow_action *action = uobj->object;
int ret;
struct ib_flow_action_esp_attr esp_attr = {};
if (!action->device->modify_flow_action_esp)
return -EOPNOTSUPP;
ret = parse_flow_action_esp(action->device, attrs, &esp_attr, true);
if (ret)
return ret;
if (action->type != IB_FLOW_ACTION_ESP)
return -EINVAL;
return action->device->modify_flow_action_esp(action,
&esp_attr.hdr,
attrs);
}
static const struct uverbs_attr_spec uverbs_flow_action_esp_keymat[] = {
[IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_STRUCT(
struct ib_uverbs_flow_action_esp_keymat_aes_gcm,
aes_key),
},
};
static const struct uverbs_attr_spec uverbs_flow_action_esp_replay[] = {
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_NO_DATA(),
},
[IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP] = {
.type = UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp_replay_bmp,
size),
},
};
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,
UVERBS_ATTR_IDR(UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,
hard_limit_pkts),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,
UVERBS_ATTR_TYPE(__u32),
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,
uverbs_flow_action_esp_keymat,
UA_MANDATORY),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,
uverbs_flow_action_esp_replay,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,
UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),
UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY,
UVERBS_ATTR_IDR(UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_WRITE,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,
UVERBS_ATTR_STRUCT(struct ib_uverbs_flow_action_esp,
hard_limit_pkts),
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_FLOW_ACTION_ESP_ESN,
UVERBS_ATTR_TYPE(__u32),
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,
uverbs_flow_action_esp_keymat,
UA_OPTIONAL),
UVERBS_ATTR_ENUM_IN(UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,
uverbs_flow_action_esp_replay,
UA_OPTIONAL),
UVERBS_ATTR_PTR_IN(
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,
UVERBS_ATTR_TYPE(struct ib_uverbs_flow_action_esp_encap),
UA_OPTIONAL));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_FLOW_ACTION_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_FLOW_ACTION_HANDLE,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_flow_action),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_CREATE),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY));
const struct uapi_definition uverbs_def_obj_flow_action[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
UVERBS_OBJECT_FLOW_ACTION,
UAPI_DEF_OBJ_NEEDS_FN(destroy_flow_action)),
{}
};

221
sys/ofed/drivers/infiniband/core/ib_uverbs_std_types_mr.c Normal file
View File

@ -0,0 +1,221 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "rdma_core.h"
#include "uverbs.h"
#include <rdma/uverbs_std_types.h>
static int uverbs_free_mr(struct ib_uobject *uobject,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs)
{
return ib_dereg_mr_user((struct ib_mr *)uobject->object,
&attrs->driver_udata);
}
static int UVERBS_HANDLER(UVERBS_METHOD_ADVISE_MR)(
struct uverbs_attr_bundle *attrs)
{
struct ib_pd *pd =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_ADVISE_MR_PD_HANDLE);
enum ib_uverbs_advise_mr_advice advice;
struct ib_device *ib_dev = pd->device;
const struct ib_sge *sg_list;
int num_sge;
u32 flags;
int ret;
/* FIXME: Extend the UAPI_DEF_OBJ_NEEDS_FN stuff.. */
if (!ib_dev->advise_mr)
return -EOPNOTSUPP;
ret = uverbs_get_const(&advice, attrs, UVERBS_ATTR_ADVISE_MR_ADVICE);
if (ret)
return ret;
ret = uverbs_get_flags32(&flags, attrs, UVERBS_ATTR_ADVISE_MR_FLAGS,
IB_UVERBS_ADVISE_MR_FLAG_FLUSH);
if (ret)
return ret;
num_sge = uverbs_attr_ptr_get_array_size(
attrs, UVERBS_ATTR_ADVISE_MR_SGE_LIST, sizeof(struct ib_sge));
if (num_sge < 0)
return num_sge;
sg_list = uverbs_attr_get_alloced_ptr(attrs,
UVERBS_ATTR_ADVISE_MR_SGE_LIST);
return ib_dev->advise_mr(pd, advice, flags, sg_list, num_sge,
attrs);
}
static int UVERBS_HANDLER(UVERBS_METHOD_DM_MR_REG)(
struct uverbs_attr_bundle *attrs)
{
struct ib_dm_mr_attr attr = {};
struct ib_uobject *uobj =
uverbs_attr_get_uobject(attrs, UVERBS_ATTR_REG_DM_MR_HANDLE);
struct ib_dm *dm =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_DM_MR_DM_HANDLE);
struct ib_pd *pd =
uverbs_attr_get_obj(attrs, UVERBS_ATTR_REG_DM_MR_PD_HANDLE);
struct ib_device *ib_dev = pd->device;
struct ib_mr *mr;
int ret;
if (!ib_dev->reg_dm_mr)
return -EOPNOTSUPP;
ret = uverbs_copy_from(&attr.offset, attrs, UVERBS_ATTR_REG_DM_MR_OFFSET);
if (ret)
return ret;
ret = uverbs_copy_from(&attr.length, attrs,
UVERBS_ATTR_REG_DM_MR_LENGTH);
if (ret)
return ret;
ret = uverbs_get_flags32(&attr.access_flags, attrs,
UVERBS_ATTR_REG_DM_MR_ACCESS_FLAGS,
IB_ACCESS_SUPPORTED);
if (ret)
return ret;
if (!(attr.access_flags & IB_ZERO_BASED))
return -EINVAL;
ret = ib_check_mr_access(attr.access_flags);
if (ret)
return ret;
if (attr.offset > dm->length || attr.length > dm->length ||
attr.length > dm->length - attr.offset)
return -EINVAL;
mr = pd->device->reg_dm_mr(pd, dm, &attr, attrs);
if (IS_ERR(mr))
return PTR_ERR(mr);
mr->device = pd->device;
mr->pd = pd;
mr->type = IB_MR_TYPE_DM;
mr->dm = dm;
mr->uobject = uobj;
atomic_inc(&pd->usecnt);
atomic_inc(&dm->usecnt);
uobj->object = mr;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_LKEY, &mr->lkey,
sizeof(mr->lkey));
if (ret)
goto err_dereg;
ret = uverbs_copy_to(attrs, UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
&mr->rkey, sizeof(mr->rkey));
if (ret)
goto err_dereg;
return 0;
err_dereg:
ib_dereg_mr_user(mr, uverbs_get_cleared_udata(attrs));
return ret;
}
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_ADVISE_MR,
UVERBS_ATTR_IDR(UVERBS_ATTR_ADVISE_MR_PD_HANDLE,
UVERBS_OBJECT_PD,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_CONST_IN(UVERBS_ATTR_ADVISE_MR_ADVICE,
enum ib_uverbs_advise_mr_advice,
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_ADVISE_MR_FLAGS,
enum ib_uverbs_advise_mr_flag,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_ADVISE_MR_SGE_LIST,
UVERBS_ATTR_MIN_SIZE(sizeof(struct ib_uverbs_sge)),
UA_MANDATORY,
UA_ALLOC_AND_COPY));
DECLARE_UVERBS_NAMED_METHOD(
UVERBS_METHOD_DM_MR_REG,
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_HANDLE,
UVERBS_OBJECT_MR,
UVERBS_ACCESS_NEW,
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DM_MR_OFFSET,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_REG_DM_MR_LENGTH,
UVERBS_ATTR_TYPE(u64),
UA_MANDATORY),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_PD_HANDLE,
UVERBS_OBJECT_PD,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_FLAGS_IN(UVERBS_ATTR_REG_DM_MR_ACCESS_FLAGS,
enum ib_access_flags),
UVERBS_ATTR_IDR(UVERBS_ATTR_REG_DM_MR_DM_HANDLE,
UVERBS_OBJECT_DM,
UVERBS_ACCESS_READ,
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DM_MR_RESP_LKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY),
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
UVERBS_ATTR_TYPE(u32),
UA_MANDATORY));
DECLARE_UVERBS_NAMED_METHOD_DESTROY(
UVERBS_METHOD_MR_DESTROY,
UVERBS_ATTR_IDR(UVERBS_ATTR_DESTROY_MR_HANDLE,
UVERBS_OBJECT_MR,
UVERBS_ACCESS_DESTROY,
UA_MANDATORY));
DECLARE_UVERBS_NAMED_OBJECT(
UVERBS_OBJECT_MR,
UVERBS_TYPE_ALLOC_IDR(uverbs_free_mr),
&UVERBS_METHOD(UVERBS_METHOD_DM_MR_REG),
&UVERBS_METHOD(UVERBS_METHOD_MR_DESTROY),
&UVERBS_METHOD(UVERBS_METHOD_ADVISE_MR));
const struct uapi_definition uverbs_def_obj_mr[] = {
UAPI_DEF_CHAIN_OBJ_TREE_NAMED(UVERBS_OBJECT_MR,
UAPI_DEF_OBJ_NEEDS_FN(dereg_mr)),
{}
};

731
sys/ofed/drivers/infiniband/core/ib_uverbs_uapi.c Normal file
View File

@ -0,0 +1,731 @@
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*/
#include <rdma/uverbs_ioctl.h>
#include <rdma/rdma_user_ioctl.h>
#include <linux/bitops.h>
#include "rdma_core.h"
#include "uverbs.h"
static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)
{
return -EOPNOTSUPP;
}
static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
{
void *elm;
int rc;
if (key == UVERBS_API_KEY_ERR)
return ERR_PTR(-EOVERFLOW);
elm = kzalloc(alloc_size, GFP_KERNEL);
if (!elm)
return ERR_PTR(-ENOMEM);
rc = radix_tree_insert(&uapi->radix, key, elm);
if (rc) {
kfree(elm);
return ERR_PTR(rc);
}
return elm;
}
static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
size_t alloc_size, bool *exists)
{
void *elm;
elm = uapi_add_elm(uapi, key, alloc_size);
if (!IS_ERR(elm)) {
*exists = false;
return elm;
}
if (elm != ERR_PTR(-EEXIST))
return elm;
elm = radix_tree_lookup(&uapi->radix, key);
if (WARN_ON(!elm))
return ERR_PTR(-EINVAL);
*exists = true;
return elm;
}
static int uapi_create_write(struct uverbs_api *uapi,
struct ib_device *ibdev,
const struct uapi_definition *def,
u32 obj_key,
u32 *cur_method_key)
{
struct uverbs_api_write_method *method_elm;
u32 method_key = obj_key;
bool exists;
if (def->write.is_ex)
method_key |= uapi_key_write_ex_method(def->write.command_num);
else
method_key |= uapi_key_write_method(def->write.command_num);
method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
&exists);
if (IS_ERR(method_elm))
return PTR_ERR(method_elm);
if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))
return -EINVAL;
method_elm->is_ex = def->write.is_ex;
method_elm->handler = def->func_write;
if (def->write.is_ex)
method_elm->disabled = !(ibdev->uverbs_ex_cmd_mask &
BIT_ULL(def->write.command_num));
else
method_elm->disabled = !(ibdev->uverbs_cmd_mask &
BIT_ULL(def->write.command_num));
if (!def->write.is_ex && def->func_write) {
method_elm->has_udata = def->write.has_udata;
method_elm->has_resp = def->write.has_resp;
method_elm->req_size = def->write.req_size;
method_elm->resp_size = def->write.resp_size;
}
*cur_method_key = method_key;
return 0;
}
static int uapi_merge_method(struct uverbs_api *uapi,
struct uverbs_api_object *obj_elm, u32 obj_key,
const struct uverbs_method_def *method,
bool is_driver)
{
u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
struct uverbs_api_ioctl_method *method_elm;
unsigned int i;
bool exists;
if (!method->attrs)
return 0;
method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
&exists);
if (IS_ERR(method_elm))
return PTR_ERR(method_elm);
if (exists) {
/*
* This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
*/
if (WARN_ON(method->handler))
return -EINVAL;
} else {
WARN_ON(!method->handler);
rcu_assign_pointer(method_elm->handler, method->handler);
if (method->handler != uverbs_destroy_def_handler)
method_elm->driver_method = is_driver;
}
for (i = 0; i != method->num_attrs; i++) {
const struct uverbs_attr_def *attr = (*method->attrs)[i];
struct uverbs_api_attr *attr_slot;
if (!attr)
continue;
/*
* ENUM_IN contains the 'ids' pointer to the driver's .rodata,
* so if it is specified by a driver then it always makes this
* into a driver method.
*/
if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
method_elm->driver_method |= is_driver;
/*
* Like other uobject based things we only support a single
* uobject being NEW'd or DESTROY'd
*/
if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
u8 access = attr->attr.u2.objs_arr.access;
if (WARN_ON(access == UVERBS_ACCESS_NEW ||
access == UVERBS_ACCESS_DESTROY))
return -EINVAL;
}
attr_slot =
uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
sizeof(*attr_slot));
/* Attributes are not allowed to be modified by drivers */
if (IS_ERR(attr_slot))
return PTR_ERR(attr_slot);
attr_slot->spec = attr->attr;
}
return 0;
}
static int uapi_merge_obj_tree(struct uverbs_api *uapi,
const struct uverbs_object_def *obj,
bool is_driver)
{
struct uverbs_api_object *obj_elm;
unsigned int i;
u32 obj_key;
bool exists;
int rc;
obj_key = uapi_key_obj(obj->id);
obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
if (IS_ERR(obj_elm))
return PTR_ERR(obj_elm);
if (obj->type_attrs) {
if (WARN_ON(obj_elm->type_attrs))
return -EINVAL;
obj_elm->id = obj->id;
obj_elm->type_attrs = obj->type_attrs;
obj_elm->type_class = obj->type_attrs->type_class;
/*
* Today drivers are only permitted to use idr_class and
* fd_class types. We can revoke the IDR types during
* disassociation, and the FD types require the driver to use
* struct file_operations.owner to prevent the driver module
* code from unloading while the file is open. This provides
* enough safety that uverbs_uobject_fd_release() will
* continue to work. Drivers using FD are responsible to
* handle disassociation of the device on their own.
*/
if (WARN_ON(is_driver &&
obj->type_attrs->type_class != &uverbs_idr_class &&
obj->type_attrs->type_class != &uverbs_fd_class))
return -EINVAL;
}
if (!obj->methods)
return 0;
for (i = 0; i != obj->num_methods; i++) {
const struct uverbs_method_def *method = (*obj->methods)[i];
if (!method)
continue;
rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
is_driver);
if (rc)
return rc;
}
return 0;
}
static int uapi_disable_elm(struct uverbs_api *uapi,
const struct uapi_definition *def,
u32 obj_key,
u32 method_key)
{
bool exists;
if (def->scope == UAPI_SCOPE_OBJECT) {
struct uverbs_api_object *obj_elm;
obj_elm = uapi_add_get_elm(
uapi, obj_key, sizeof(*obj_elm), &exists);
if (IS_ERR(obj_elm))
return PTR_ERR(obj_elm);
obj_elm->disabled = 1;
return 0;
}
if (def->scope == UAPI_SCOPE_METHOD &&
uapi_key_is_ioctl_method(method_key)) {
struct uverbs_api_ioctl_method *method_elm;
method_elm = uapi_add_get_elm(uapi, method_key,
sizeof(*method_elm), &exists);
if (IS_ERR(method_elm))
return PTR_ERR(method_elm);
method_elm->disabled = 1;
return 0;
}
if (def->scope == UAPI_SCOPE_METHOD &&
(uapi_key_is_write_method(method_key) ||
uapi_key_is_write_ex_method(method_key))) {
struct uverbs_api_write_method *write_elm;
write_elm = uapi_add_get_elm(uapi, method_key,
sizeof(*write_elm), &exists);
if (IS_ERR(write_elm))
return PTR_ERR(write_elm);
write_elm->disabled = 1;
return 0;
}
WARN_ON(true);
return -EINVAL;
}
static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
const struct uapi_definition *def_list,
bool is_driver)
{
const struct uapi_definition *def = def_list;
u32 cur_obj_key = UVERBS_API_KEY_ERR;
u32 cur_method_key = UVERBS_API_KEY_ERR;
bool exists;
int rc;
if (!def_list)
return 0;
for (;; def++) {
switch ((enum uapi_definition_kind)def->kind) {
case UAPI_DEF_CHAIN:
rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
if (rc)
return rc;
continue;
case UAPI_DEF_CHAIN_OBJ_TREE:
if (WARN_ON(def->object_start.object_id !=
def->chain_obj_tree->id))
return -EINVAL;
cur_obj_key = uapi_key_obj(def->object_start.object_id);
rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
is_driver);
if (rc)
return rc;
continue;
case UAPI_DEF_END:
return 0;
case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
void **ibdev_fn =
(void *)((u8 *)ibdev + def->needs_fn_offset);
if (*ibdev_fn)
continue;
rc = uapi_disable_elm(
uapi, def, cur_obj_key, cur_method_key);
if (rc)
return rc;
continue;
}
case UAPI_DEF_IS_SUPPORTED_FUNC:
if (def->func_is_supported(ibdev))
continue;
rc = uapi_disable_elm(
uapi, def, cur_obj_key, cur_method_key);
if (rc)
return rc;
continue;
case UAPI_DEF_OBJECT_START: {
struct uverbs_api_object *obj_elm;
cur_obj_key = uapi_key_obj(def->object_start.object_id);
obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
sizeof(*obj_elm), &exists);
if (IS_ERR(obj_elm))
return PTR_ERR(obj_elm);
continue;
}
case UAPI_DEF_WRITE:
rc = uapi_create_write(
uapi, ibdev, def, cur_obj_key, &cur_method_key);
if (rc)
return rc;
continue;
}
WARN_ON(true);
return -EINVAL;
}
}
static int
uapi_finalize_ioctl_method(struct uverbs_api *uapi,
struct uverbs_api_ioctl_method *method_elm,
u32 method_key)
{
struct radix_tree_iter iter;
unsigned int num_attrs = 0;
unsigned int max_bkey = 0;
bool single_uobj = false;
void __rcu **slot;
method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
radix_tree_for_each_slot (slot, &uapi->radix, &iter,
uapi_key_attrs_start(method_key)) {
struct uverbs_api_attr *elm =
rcu_dereference_protected(*slot, true);
u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
u32 attr_bkey = uapi_bkey_attr(attr_key);
u8 type = elm->spec.type;
if (uapi_key_attr_to_ioctl_method(iter.index) !=
uapi_key_attr_to_ioctl_method(method_key))
break;
if (elm->spec.mandatory)
__set_bit(attr_bkey, method_elm->attr_mandatory);
if (elm->spec.is_udata)
method_elm->has_udata = true;
if (type == UVERBS_ATTR_TYPE_IDR ||
type == UVERBS_ATTR_TYPE_FD) {
u8 access = elm->spec.u.obj.access;
/*
* Verbs specs may only have one NEW/DESTROY, we don't
* have the infrastructure to abort multiple NEW's or
* cope with multiple DESTROY failure.
*/
if (access == UVERBS_ACCESS_NEW ||
access == UVERBS_ACCESS_DESTROY) {
if (WARN_ON(single_uobj))
return -EINVAL;
single_uobj = true;
if (WARN_ON(!elm->spec.mandatory))
return -EINVAL;
}
if (access == UVERBS_ACCESS_DESTROY)
method_elm->destroy_bkey = attr_bkey;
}
max_bkey = max(max_bkey, attr_bkey);
num_attrs++;
}
method_elm->key_bitmap_len = max_bkey + 1;
WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
uapi_compute_bundle_size(method_elm, num_attrs);
return 0;
}
static int uapi_finalize(struct uverbs_api *uapi)
{
const struct uverbs_api_write_method **data;
unsigned long max_write_ex = 0;
unsigned long max_write = 0;
struct radix_tree_iter iter;
void __rcu **slot;
int rc;
int i;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
struct uverbs_api_ioctl_method *method_elm =
rcu_dereference_protected(*slot, true);
if (uapi_key_is_ioctl_method(iter.index)) {
rc = uapi_finalize_ioctl_method(uapi, method_elm,
iter.index);
if (rc)
return rc;
}
if (uapi_key_is_write_method(iter.index))
max_write = max(max_write,
iter.index & UVERBS_API_ATTR_KEY_MASK);
if (uapi_key_is_write_ex_method(iter.index))
max_write_ex =
max(max_write_ex,
iter.index & UVERBS_API_ATTR_KEY_MASK);
}
uapi->notsupp_method.handler = ib_uverbs_notsupp;
uapi->num_write = max_write + 1;
uapi->num_write_ex = max_write_ex + 1;
data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
sizeof(*uapi->write_methods), GFP_KERNEL);
for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
data[i] = &uapi->notsupp_method;
uapi->write_methods = data;
uapi->write_ex_methods = data + uapi->num_write;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
if (uapi_key_is_write_method(iter.index))
uapi->write_methods[iter.index &
UVERBS_API_ATTR_KEY_MASK] =
rcu_dereference_protected(*slot, true);
if (uapi_key_is_write_ex_method(iter.index))
uapi->write_ex_methods[iter.index &
UVERBS_API_ATTR_KEY_MASK] =
rcu_dereference_protected(*slot, true);
}
return 0;
}
static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
{
struct radix_tree_iter iter;
void __rcu **slot;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
if (iter.index > last)
return;
kfree(rcu_dereference_protected(*slot, true));
radix_tree_iter_delete(&uapi->radix, &iter, slot);
}
}
static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
{
uapi_remove_range(uapi, obj_key,
obj_key | UVERBS_API_METHOD_KEY_MASK |
UVERBS_API_ATTR_KEY_MASK);
}
static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
{
uapi_remove_range(uapi, method_key,
method_key | UVERBS_API_ATTR_KEY_MASK);
}
static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
{
if (spec->type == UVERBS_ATTR_TYPE_IDR ||
spec->type == UVERBS_ATTR_TYPE_FD)
return spec->u.obj.obj_type;
if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
return spec->u2.objs_arr.obj_type;
return UVERBS_API_KEY_ERR;
}
static void uapi_key_okay(u32 key)
{
unsigned int count = 0;
if (uapi_key_is_object(key))
count++;
if (uapi_key_is_ioctl_method(key))
count++;
if (uapi_key_is_write_method(key))
count++;
if (uapi_key_is_write_ex_method(key))
count++;
if (uapi_key_is_attr(key))
count++;
WARN(count != 1, "Bad count %d key=%x", count, key);
}
static void uapi_finalize_disable(struct uverbs_api *uapi)
{
struct radix_tree_iter iter;
u32 starting_key = 0;
bool scan_again = false;
void __rcu **slot;
again:
radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
uapi_key_okay(iter.index);
if (uapi_key_is_object(iter.index)) {
struct uverbs_api_object *obj_elm =
rcu_dereference_protected(*slot, true);
if (obj_elm->disabled) {
/* Have to check all the attrs again */
scan_again = true;
starting_key = iter.index;
uapi_remove_object(uapi, iter.index);
goto again;
}
continue;
}
if (uapi_key_is_ioctl_method(iter.index)) {
struct uverbs_api_ioctl_method *method_elm =
rcu_dereference_protected(*slot, true);
if (method_elm->disabled) {
starting_key = iter.index;
uapi_remove_method(uapi, iter.index);
goto again;
}
continue;
}
if (uapi_key_is_write_method(iter.index) ||
uapi_key_is_write_ex_method(iter.index)) {
struct uverbs_api_write_method *method_elm =
rcu_dereference_protected(*slot, true);
if (method_elm->disabled) {
kfree(method_elm);
radix_tree_iter_delete(&uapi->radix, &iter, slot);
}
continue;
}
if (uapi_key_is_attr(iter.index)) {
struct uverbs_api_attr *attr_elm =
rcu_dereference_protected(*slot, true);
const struct uverbs_api_object *tmp_obj;
u32 obj_key;
/*
* If the method has a mandatory object handle
* attribute which relies on an object which is not
* present then the entire method is uncallable.
*/
if (!attr_elm->spec.mandatory)
continue;
obj_key = uapi_get_obj_id(&attr_elm->spec);
if (obj_key == UVERBS_API_KEY_ERR)
continue;
tmp_obj = uapi_get_object(uapi, obj_key);
if (IS_ERR(tmp_obj)) {
if (PTR_ERR(tmp_obj) == -ENOMSG)
continue;
} else {
if (!tmp_obj->disabled)
continue;
}
starting_key = iter.index;
uapi_remove_method(
uapi,
iter.index & (UVERBS_API_OBJ_KEY_MASK |
UVERBS_API_METHOD_KEY_MASK));
goto again;
}
WARN_ON(false);
}
if (!scan_again)
return;
scan_again = false;
starting_key = 0;
goto again;
}
void uverbs_destroy_api(struct uverbs_api *uapi)
{
if (!uapi)
return;
uapi_remove_range(uapi, 0, U32_MAX);
kfree(uapi->write_methods);
kfree(uapi);
}
static const struct uapi_definition uverbs_core_api[] = {
UAPI_DEF_CHAIN(uverbs_def_obj_async_fd),
UAPI_DEF_CHAIN(uverbs_def_obj_counters),
UAPI_DEF_CHAIN(uverbs_def_obj_cq),
UAPI_DEF_CHAIN(uverbs_def_obj_device),
UAPI_DEF_CHAIN(uverbs_def_obj_dm),
UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
UAPI_DEF_CHAIN(uverbs_def_obj_intf),
UAPI_DEF_CHAIN(uverbs_def_obj_mr),
UAPI_DEF_CHAIN(uverbs_def_write_intf),
{},
};
struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
{
struct uverbs_api *uapi;
int rc;
uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
if (!uapi)
return ERR_PTR(-ENOMEM);
INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
uapi->driver_id = ibdev->ops.driver_id;
rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
if (rc)
goto err;
rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
if (rc)
goto err;
uapi_finalize_disable(uapi);
rc = uapi_finalize(uapi);
if (rc)
goto err;
return uapi;
err:
if (rc != -ENOMEM)
dev_err(&ibdev->dev,
"Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
rc);
uverbs_destroy_api(uapi);
return ERR_PTR(rc);
}
/*
* The pre version is done before destroying the HW objects, it only blocks
* off method access. All methods that require the ib_dev or the module data
* must test one of these assignments prior to continuing.
*/
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
{
struct uverbs_api *uapi = uverbs_dev->uapi;
struct radix_tree_iter iter;
void __rcu **slot;
rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
if (uapi_key_is_ioctl_method(iter.index)) {
struct uverbs_api_ioctl_method *method_elm =
rcu_dereference_protected(*slot, true);
if (method_elm->driver_method)
rcu_assign_pointer(method_elm->handler, NULL);
}
}
synchronize_srcu(&uverbs_dev->disassociate_srcu);
}
/*
* Called when a driver disassociates from the ib_uverbs_device. The
* assumption is that the driver module will unload after. Replace everything
* related to the driver with NULL as a safety measure.
*/
void uverbs_disassociate_api(struct uverbs_api *uapi)
{
struct radix_tree_iter iter;
void __rcu **slot;
radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
if (uapi_key_is_object(iter.index)) {
struct uverbs_api_object *object_elm =
rcu_dereference_protected(*slot, true);
/*
* Some type_attrs are in the driver module. We don't
* bother to keep track of which since there should be
* no use of this after disassociate.
*/
object_elm->type_attrs = NULL;
} else if (uapi_key_is_attr(iter.index)) {
struct uverbs_api_attr *elm =
rcu_dereference_protected(*slot, true);
if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
elm->spec.u2.enum_def.ids = NULL;
}
}
}

455
sys/ofed/drivers/infiniband/core/ib_verbs.c
View File

@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
#include <linux/slab.h>
#include <linux/in.h>
#include <linux/in6.h>
#include <linux/wait.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_cache.h>
@ -267,10 +268,11 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
{
struct ib_pd *pd;
int mr_access_flags = 0;
int ret;
pd = device->alloc_pd(device, NULL, NULL);
if (IS_ERR(pd))
return pd;
pd = rdma_zalloc_drv_obj(device, ib_pd);
if (!pd)
return ERR_PTR(-ENOMEM);
pd->device = device;
pd->uobject = NULL;
@ -278,6 +280,12 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
atomic_set(&pd->usecnt, 0);
pd->flags = flags;
ret = device->alloc_pd(pd, NULL);
if (ret) {
kfree(pd);
return ERR_PTR(ret);
}
if (device->attrs.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY)
pd->local_dma_lkey = device->local_dma_lkey;
else
@ -299,6 +307,7 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
mr->device = pd->device;
mr->pd = pd;
mr->type = IB_MR_TYPE_DMA;
mr->uobject = NULL;
mr->need_inval = false;
@ -316,19 +325,20 @@ struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
EXPORT_SYMBOL(__ib_alloc_pd);
/**
* ib_dealloc_pd - Deallocates a protection domain.
* ib_dealloc_pd_user - Deallocates a protection domain.
* @pd: The protection domain to deallocate.
* @udata: Valid user data or NULL for kernel object
*
* It is an error to call this function while any resources in the pd still
* exist. The caller is responsible to synchronously destroy them and
* guarantee no new allocations will happen.
*/
void ib_dealloc_pd(struct ib_pd *pd)
void ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
{
int ret;
if (pd->__internal_mr) {
ret = pd->device->dereg_mr(pd->__internal_mr);
ret = pd->device->dereg_mr(pd->__internal_mr, NULL);
WARN_ON(ret);
pd->__internal_mr = NULL;
}
@ -337,32 +347,98 @@ void ib_dealloc_pd(struct ib_pd *pd)
requires the caller to guarantee we can't race here. */
WARN_ON(atomic_read(&pd->usecnt));
/* Making delalloc_pd a void return is a WIP, no driver should return
an error here. */
ret = pd->device->dealloc_pd(pd);
WARN_ONCE(ret, "Infiniband HW driver failed dealloc_pd");
pd->device->dealloc_pd(pd, udata);
kfree(pd);
}
EXPORT_SYMBOL(ib_dealloc_pd);
EXPORT_SYMBOL(ib_dealloc_pd_user);
/* Address handles */
struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
static struct ib_ah *_ib_create_ah(struct ib_pd *pd,
struct ib_ah_attr *ah_attr,
u32 flags,
struct ib_udata *udata)
{
struct ib_device *device = pd->device;
struct ib_ah *ah;
int ret;
might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
if (!device->create_ah)
return ERR_PTR(-EOPNOTSUPP);
ah = rdma_zalloc_drv_obj_gfp(
device, ib_ah,
(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
if (!ah)
return ERR_PTR(-ENOMEM);
ah->device = device;
ah->pd = pd;
ret = device->create_ah(ah, ah_attr, flags, udata);
if (ret) {
kfree(ah);
return ERR_PTR(ret);
}
atomic_inc(&pd->usecnt);
return ah;
}
/**
* rdma_create_ah - Creates an address handle for the
* given address vector.
* @pd: The protection domain associated with the address handle.
* @ah_attr: The attributes of the address vector.
* @flags: Create address handle flags (see enum rdma_create_ah_flags).
*
* It returns 0 on success and returns appropriate error code on error.
* The address handle is used to reference a local or global destination
* in all UD QP post sends.
*/
struct ib_ah *ib_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr,
u32 flags)
{
struct ib_ah *ah;
ah = pd->device->create_ah(pd, ah_attr, NULL);
if (!IS_ERR(ah)) {
ah->device = pd->device;
ah->pd = pd;
ah->uobject = NULL;
atomic_inc(&pd->usecnt);
}
ah = _ib_create_ah(pd, ah_attr, flags, NULL);
return ah;
}
EXPORT_SYMBOL(ib_create_ah);
/**
* ib_create_user_ah - Creates an address handle for the
* given address vector.
* It resolves destination mac address for ah attribute of RoCE type.
* @pd: The protection domain associated with the address handle.
* @ah_attr: The attributes of the address vector.
* @udata: pointer to user's input output buffer information need by
* provider driver.
*
* It returns a valid address handle pointer on success and
* returns appropriate error code on error.
* The address handle is used to reference a local or global destination
* in all UD QP post sends.
*/
struct ib_ah *ib_create_user_ah(struct ib_pd *pd,
struct ib_ah_attr *ah_attr,
struct ib_udata *udata)
{
int err;
if (rdma_protocol_roce(pd->device, ah_attr->port_num)) {
err = ib_resolve_eth_dmac(pd->device, ah_attr);
if (err)
return ERR_PTR(err);
}
return _ib_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE, udata);
}
EXPORT_SYMBOL(ib_create_user_ah);
static int ib_get_header_version(const union rdma_network_hdr *hdr)
{
const struct ip *ip4h = (const struct ip *)&hdr->roce4grh;
@ -589,7 +665,7 @@ struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
if (ret)
return ERR_PTR(ret);
return ib_create_ah(pd, &ah_attr);
return ib_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
}
EXPORT_SYMBOL(ib_create_ah_from_wc);
@ -609,19 +685,20 @@ int ib_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
}
EXPORT_SYMBOL(ib_query_ah);
int ib_destroy_ah(struct ib_ah *ah)
int ib_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
{
struct ib_pd *pd;
int ret;
might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
pd = ah->pd;
ret = ah->device->destroy_ah(ah);
if (!ret)
atomic_dec(&pd->usecnt);
ah->device->destroy_ah(ah, flags);
atomic_dec(&pd->usecnt);
return ret;
kfree(ah);
return 0;
}
EXPORT_SYMBOL(ib_destroy_ah);
EXPORT_SYMBOL(ib_destroy_ah_user);
/* Shared receive queues */
@ -629,27 +706,40 @@ struct ib_srq *ib_create_srq(struct ib_pd *pd,
struct ib_srq_init_attr *srq_init_attr)
{
struct ib_srq *srq;
int ret;
if (!pd->device->create_srq)
return ERR_PTR(-ENOSYS);
return ERR_PTR(-EOPNOTSUPP);
srq = pd->device->create_srq(pd, srq_init_attr, NULL);
srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
if (!srq)
return ERR_PTR(-ENOMEM);
if (!IS_ERR(srq)) {
srq->device = pd->device;
srq->pd = pd;
srq->uobject = NULL;
srq->event_handler = srq_init_attr->event_handler;
srq->srq_context = srq_init_attr->srq_context;
srq->srq_type = srq_init_attr->srq_type;
if (srq->srq_type == IB_SRQT_XRC) {
srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
srq->ext.xrc.cq = srq_init_attr->ext.xrc.cq;
atomic_inc(&srq->ext.xrc.xrcd->usecnt);
atomic_inc(&srq->ext.xrc.cq->usecnt);
}
atomic_inc(&pd->usecnt);
atomic_set(&srq->usecnt, 0);
srq->device = pd->device;
srq->pd = pd;
srq->event_handler = srq_init_attr->event_handler;
srq->srq_context = srq_init_attr->srq_context;
srq->srq_type = srq_init_attr->srq_type;
if (ib_srq_has_cq(srq->srq_type)) {
srq->ext.cq = srq_init_attr->ext.cq;
atomic_inc(&srq->ext.cq->usecnt);
}
if (srq->srq_type == IB_SRQT_XRC) {
srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
atomic_inc(&srq->ext.xrc.xrcd->usecnt);
}
atomic_inc(&pd->usecnt);
ret = pd->device->create_srq(srq, srq_init_attr, NULL);
if (ret) {
atomic_dec(&srq->pd->usecnt);
if (srq->srq_type == IB_SRQT_XRC)
atomic_dec(&srq->ext.xrc.xrcd->usecnt);
if (ib_srq_has_cq(srq->srq_type))
atomic_dec(&srq->ext.cq->usecnt);
kfree(srq);
return ERR_PTR(ret);
}
return srq;
@ -674,36 +764,23 @@ int ib_query_srq(struct ib_srq *srq,
}
EXPORT_SYMBOL(ib_query_srq);
int ib_destroy_srq(struct ib_srq *srq)
int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
{
struct ib_pd *pd;
enum ib_srq_type srq_type;
struct ib_xrcd *uninitialized_var(xrcd);
struct ib_cq *uninitialized_var(cq);
int ret;
if (atomic_read(&srq->usecnt))
return -EBUSY;
pd = srq->pd;
srq_type = srq->srq_type;
if (srq_type == IB_SRQT_XRC) {
xrcd = srq->ext.xrc.xrcd;
cq = srq->ext.xrc.cq;
}
srq->device->destroy_srq(srq, udata);
ret = srq->device->destroy_srq(srq);
if (!ret) {
atomic_dec(&pd->usecnt);
if (srq_type == IB_SRQT_XRC) {
atomic_dec(&xrcd->usecnt);
atomic_dec(&cq->usecnt);
}
}
atomic_dec(&srq->pd->usecnt);
if (srq->srq_type == IB_SRQT_XRC)
atomic_dec(&srq->ext.xrc.xrcd->usecnt);
if (ib_srq_has_cq(srq->srq_type))
atomic_dec(&srq->ext.cq->usecnt);
kfree(srq);
return ret;
return 0;
}
EXPORT_SYMBOL(ib_destroy_srq);
EXPORT_SYMBOL(ib_destroy_srq_user);
/* Queue pairs */
@ -793,7 +870,7 @@ static struct ib_qp *ib_create_xrc_qp(struct ib_qp *qp,
if (!IS_ERR(qp))
__ib_insert_xrcd_qp(qp_init_attr->xrcd, real_qp);
else
real_qp->device->destroy_qp(real_qp);
real_qp->device->destroy_qp(real_qp, NULL);
return qp;
}
@ -809,7 +886,7 @@ struct ib_qp *ib_create_qp(struct ib_pd *pd,
qp_init_attr->cap.max_recv_sge))
return ERR_PTR(-EINVAL);
qp = device->create_qp(pd, qp_init_attr, NULL);
qp = _ib_create_qp(device, pd, qp_init_attr, NULL, NULL);
if (IS_ERR(qp))
return qp;
@ -1243,6 +1320,95 @@ int ib_resolve_eth_dmac(struct ib_device *device,
}
EXPORT_SYMBOL(ib_resolve_eth_dmac);
static bool is_qp_type_connected(const struct ib_qp *qp)
{
return (qp->qp_type == IB_QPT_UC ||
qp->qp_type == IB_QPT_RC ||
qp->qp_type == IB_QPT_XRC_INI ||
qp->qp_type == IB_QPT_XRC_TGT);
}
/**
* IB core internal function to perform QP attributes modification.
*/
static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
u8 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
int ret;
if (port < rdma_start_port(qp->device) ||
port > rdma_end_port(qp->device))
return -EINVAL;
if (attr_mask & IB_QP_ALT_PATH) {
/*
* Today the core code can only handle alternate paths and APM
* for IB. Ban them in roce mode.
*/
if (!(rdma_protocol_ib(qp->device,
attr->alt_ah_attr.port_num) &&
rdma_protocol_ib(qp->device, port))) {
ret = EINVAL;
goto out;
}
}
/*
* If the user provided the qp_attr then we have to resolve it. Kernel
* users have to provide already resolved rdma_ah_attr's
*/
if (udata && (attr_mask & IB_QP_AV) &&
rdma_protocol_roce(qp->device, port) &&
is_qp_type_connected(qp)) {
ret = ib_resolve_eth_dmac(qp->device, &attr->ah_attr);
if (ret)
goto out;
}
if (rdma_ib_or_roce(qp->device, port)) {
if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
dev_warn(&qp->device->dev,
"%s rq_psn overflow, masking to 24 bits\n",
__func__);
attr->rq_psn &= 0xffffff;
}
if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
dev_warn(&qp->device->dev,
" %s sq_psn overflow, masking to 24 bits\n",
__func__);
attr->sq_psn &= 0xffffff;
}
}
ret = qp->device->modify_qp(qp, attr, attr_mask, udata);
if (ret)
goto out;
if (attr_mask & IB_QP_PORT)
qp->port = attr->port_num;
out:
return ret;
}
/**
* ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
* @ib_qp: The QP to modify.
* @attr: On input, specifies the QP attributes to modify. On output,
* the current values of selected QP attributes are returned.
* @attr_mask: A bit-mask used to specify which attributes of the QP
* are being modified.
* @udata: pointer to user's input output buffer information
* are being modified.
* It returns 0 on success and returns appropriate error code on error.
*/
int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
int attr_mask, struct ib_udata *udata)
{
return _ib_modify_qp(ib_qp->real_qp, attr, attr_mask, udata);
}
EXPORT_SYMBOL(ib_modify_qp_with_udata);
int ib_modify_qp(struct ib_qp *qp,
struct ib_qp_attr *qp_attr,
@ -1319,7 +1485,7 @@ static int __ib_destroy_shared_qp(struct ib_qp *qp)
return 0;
}
int ib_destroy_qp(struct ib_qp *qp)
int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
{
struct ib_pd *pd;
struct ib_cq *scq, *rcq;
@ -1339,7 +1505,7 @@ int ib_destroy_qp(struct ib_qp *qp)
srq = qp->srq;
ind_tbl = qp->rwq_ind_tbl;
ret = qp->device->destroy_qp(qp);
ret = qp->device->destroy_qp(qp, udata);
if (!ret) {
if (pd)
atomic_dec(&pd->usecnt);
@ -1355,32 +1521,40 @@ int ib_destroy_qp(struct ib_qp *qp)
return ret;
}
EXPORT_SYMBOL(ib_destroy_qp);
EXPORT_SYMBOL(ib_destroy_qp_user);
/* Completion queues */
struct ib_cq *ib_create_cq(struct ib_device *device,
ib_comp_handler comp_handler,
void (*event_handler)(struct ib_event *, void *),
void *cq_context,
const struct ib_cq_init_attr *cq_attr)
struct ib_cq *__ib_create_cq(struct ib_device *device,
ib_comp_handler comp_handler,
void (*event_handler)(struct ib_event *, void *),
void *cq_context,
const struct ib_cq_init_attr *cq_attr,
const char *caller)
{
struct ib_cq *cq;
int ret;
cq = device->create_cq(device, cq_attr, NULL, NULL);
cq = rdma_zalloc_drv_obj(device, ib_cq);
if (!cq)
return ERR_PTR(-ENOMEM);
if (!IS_ERR(cq)) {
cq->device = device;
cq->uobject = NULL;
cq->comp_handler = comp_handler;
cq->event_handler = event_handler;
cq->cq_context = cq_context;
atomic_set(&cq->usecnt, 0);
cq->device = device;
cq->uobject = NULL;
cq->comp_handler = comp_handler;
cq->event_handler = event_handler;
cq->cq_context = cq_context;
atomic_set(&cq->usecnt, 0);
ret = device->create_cq(cq, cq_attr, NULL);
if (ret) {
kfree(cq);
return ERR_PTR(ret);
}
return cq;
}
EXPORT_SYMBOL(ib_create_cq);
EXPORT_SYMBOL(__ib_create_cq);
int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
{
@ -1389,14 +1563,16 @@ int ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period)
}
EXPORT_SYMBOL(ib_modify_cq);
int ib_destroy_cq(struct ib_cq *cq)
int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
{
if (atomic_read(&cq->usecnt))
return -EBUSY;
return cq->device->destroy_cq(cq);
cq->device->destroy_cq(cq, udata);
kfree(cq);
return 0;
}
EXPORT_SYMBOL(ib_destroy_cq);
EXPORT_SYMBOL(ib_destroy_cq_user);
int ib_resize_cq(struct ib_cq *cq, int cqe)
{
@ -1407,24 +1583,31 @@ EXPORT_SYMBOL(ib_resize_cq);
/* Memory regions */
int ib_dereg_mr(struct ib_mr *mr)
int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
{
struct ib_pd *pd = mr->pd;
struct ib_dm *dm = mr->dm;
struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
int ret;
ret = mr->device->dereg_mr(mr);
if (!ret)
ret = mr->device->dereg_mr(mr, udata);
if (!ret) {
atomic_dec(&pd->usecnt);
if (dm)
atomic_dec(&dm->usecnt);
kfree(sig_attrs);
}
return ret;
}
EXPORT_SYMBOL(ib_dereg_mr);
EXPORT_SYMBOL(ib_dereg_mr_user);
/**
* ib_alloc_mr() - Allocates a memory region
* ib_alloc_mr_user() - Allocates a memory region
* @pd: protection domain associated with the region
* @mr_type: memory region type
* @max_num_sg: maximum sg entries available for registration.
* @udata: user data or null for kernel objects
*
* Notes:
* Memory registeration page/sg lists must not exceed max_num_sg.
@ -1432,27 +1615,38 @@ EXPORT_SYMBOL(ib_dereg_mr);
* max_num_sg * used_page_size.
*
*/
struct ib_mr *ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg)
struct ib_mr *ib_alloc_mr_user(struct ib_pd *pd, enum ib_mr_type mr_type,
u32 max_num_sg, struct ib_udata *udata)
{
struct ib_mr *mr;
if (!pd->device->alloc_mr)
return ERR_PTR(-ENOSYS);
if (!pd->device->alloc_mr) {
mr = ERR_PTR(-EOPNOTSUPP);
goto out;
}
mr = pd->device->alloc_mr(pd, mr_type, max_num_sg);
if (mr_type == IB_MR_TYPE_INTEGRITY) {
WARN_ON_ONCE(1);
mr = ERR_PTR(-EINVAL);
goto out;
}
mr = pd->device->alloc_mr(pd, mr_type, max_num_sg, udata);
if (!IS_ERR(mr)) {
mr->device = pd->device;
mr->pd = pd;
mr->dm = NULL;
mr->uobject = NULL;
atomic_inc(&pd->usecnt);
mr->need_inval = false;
mr->type = mr_type;
mr->sig_attrs = NULL;
}
out:
return mr;
}
EXPORT_SYMBOL(ib_alloc_mr);
EXPORT_SYMBOL(ib_alloc_mr_user);
/* "Fast" memory regions */
@ -1578,14 +1772,14 @@ int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
}
EXPORT_SYMBOL(ib_detach_mcast);
struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
struct ib_xrcd *__ib_alloc_xrcd(struct ib_device *device, const char *caller)
{
struct ib_xrcd *xrcd;
if (!device->alloc_xrcd)
return ERR_PTR(-ENOSYS);
return ERR_PTR(-EOPNOTSUPP);
xrcd = device->alloc_xrcd(device, NULL, NULL);
xrcd = device->alloc_xrcd(device, NULL);
if (!IS_ERR(xrcd)) {
xrcd->device = device;
xrcd->inode = NULL;
@ -1596,9 +1790,9 @@ struct ib_xrcd *ib_alloc_xrcd(struct ib_device *device)
return xrcd;
}
EXPORT_SYMBOL(ib_alloc_xrcd);
EXPORT_SYMBOL(__ib_alloc_xrcd);
int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
int ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
{
struct ib_qp *qp;
int ret;
@ -1612,8 +1806,9 @@ int ib_dealloc_xrcd(struct ib_xrcd *xrcd)
if (ret)
return ret;
}
mutex_destroy(&xrcd->tgt_qp_mutex);
return xrcd->device->dealloc_xrcd(xrcd);
return xrcd->device->dealloc_xrcd(xrcd, udata);
}
EXPORT_SYMBOL(ib_dealloc_xrcd);
@ -1657,24 +1852,23 @@ struct ib_wq *ib_create_wq(struct ib_pd *pd,
EXPORT_SYMBOL(ib_create_wq);
/**
* ib_destroy_wq - Destroys the specified WQ.
* ib_destroy_wq - Destroys the specified user WQ.
* @wq: The WQ to destroy.
* @udata: Valid user data
*/
int ib_destroy_wq(struct ib_wq *wq)
int ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
{
int err;
struct ib_cq *cq = wq->cq;
struct ib_pd *pd = wq->pd;
if (atomic_read(&wq->usecnt))
return -EBUSY;
err = wq->device->destroy_wq(wq);
if (!err) {
atomic_dec(&pd->usecnt);
atomic_dec(&cq->usecnt);
}
return err;
wq->device->destroy_wq(wq, udata);
atomic_dec(&pd->usecnt);
atomic_dec(&cq->usecnt);
return 0;
}
EXPORT_SYMBOL(ib_destroy_wq);
@ -1761,33 +1955,6 @@ int ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *rwq_ind_table)
}
EXPORT_SYMBOL(ib_destroy_rwq_ind_table);
struct ib_flow *ib_create_flow(struct ib_qp *qp,
struct ib_flow_attr *flow_attr,
int domain)
{
struct ib_flow *flow_id;
if (!qp->device->create_flow)
return ERR_PTR(-ENOSYS);
flow_id = qp->device->create_flow(qp, flow_attr, domain);
if (!IS_ERR(flow_id))
atomic_inc(&qp->usecnt);
return flow_id;
}
EXPORT_SYMBOL(ib_create_flow);
int ib_destroy_flow(struct ib_flow *flow_id)
{
int err;
struct ib_qp *qp = flow_id->qp;
err = qp->device->destroy_flow(flow_id);
if (!err)
atomic_dec(&qp->usecnt);
return err;
}
EXPORT_SYMBOL(ib_destroy_flow);
int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
struct ib_mr_status *mr_status)
{

189
sys/ofed/drivers/infiniband/core/rdma_core.h Normal file
View File

@ -0,0 +1,189 @@
/*
* Copyright (c) 2005 Topspin Communications. All rights reserved.
* Copyright (c) 2005, 2006 Cisco Systems. All rights reserved.
* Copyright (c) 2005-2017 Mellanox Technologies. All rights reserved.
* Copyright (c) 2005 Voltaire, Inc. All rights reserved.
* Copyright (c) 2005 PathScale, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef RDMA_CORE_H
#define RDMA_CORE_H
#include <linux/idr.h>
#include <linux/radix-tree.h>
#include <rdma/uverbs_types.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/ib_verbs.h>
#include <linux/mutex.h>
struct ib_uverbs_device;
void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
enum rdma_remove_reason reason);
int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs);
/*
* Get an ib_uobject that corresponds to the given id from ufile, assuming
* the object is from the given type. Lock it to the required access when
* applicable.
* This function could create (access == NEW), destroy (access == DESTROY)
* or unlock (access == READ || access == WRITE) objects if required.
* The action will be finalized only when uverbs_finalize_object or
* uverbs_finalize_objects are called.
*/
struct ib_uobject *
uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
s64 id, struct uverbs_attr_bundle *attrs);
void uverbs_finalize_object(struct ib_uobject *uobj,
enum uverbs_obj_access access, bool commit,
struct uverbs_attr_bundle *attrs);
int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx);
void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile);
void release_ufile_idr_uobject(struct ib_uverbs_file *ufile);
struct ib_udata *uverbs_get_cleared_udata(struct uverbs_attr_bundle *attrs);
/*
* This is the runtime description of the uverbs API, used by the syscall
* machinery to validate and dispatch calls.
*/
/*
* Depending on ID the slot pointer in the radix tree points at one of these
* structs.
*/
struct uverbs_api_ioctl_method {
int(__rcu *handler)(struct uverbs_attr_bundle *attrs);
DECLARE_BITMAP(attr_mandatory, UVERBS_API_ATTR_BKEY_LEN);
u16 bundle_size;
u8 use_stack:1;
u8 driver_method:1;
u8 disabled:1;
u8 has_udata:1;
u8 key_bitmap_len;
u8 destroy_bkey;
};
struct uverbs_api_write_method {
int (*handler)(struct uverbs_attr_bundle *attrs);
u8 disabled:1;
u8 is_ex:1;
u8 has_udata:1;
u8 has_resp:1;
u8 req_size;
u8 resp_size;
};
struct uverbs_api_attr {
struct uverbs_attr_spec spec;
};
struct uverbs_api {
/* radix tree contains struct uverbs_api_* pointers */
struct radix_tree_root radix;
enum rdma_driver_id driver_id;
unsigned int num_write;
unsigned int num_write_ex;
struct uverbs_api_write_method notsupp_method;
const struct uverbs_api_write_method **write_methods;
const struct uverbs_api_write_method **write_ex_methods;
};
/*
* Get an uverbs_api_object that corresponds to the given object_id.
* Note:
* -ENOMSG means that any object is allowed to match during lookup.
*/
static inline const struct uverbs_api_object *
uapi_get_object(struct uverbs_api *uapi, u16 object_id)
{
const struct uverbs_api_object *res;
if (object_id == UVERBS_IDR_ANY_OBJECT)
return ERR_PTR(-ENOMSG);
res = radix_tree_lookup(&uapi->radix, uapi_key_obj(object_id));
if (!res)
return ERR_PTR(-ENOENT);
return res;
}
char *uapi_key_format(char *S, unsigned int key);
struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev);
void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev);
void uverbs_disassociate_api(struct uverbs_api *uapi);
void uverbs_destroy_api(struct uverbs_api *uapi);
void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
unsigned int num_attrs);
void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile);
extern const struct uapi_definition uverbs_def_obj_async_fd[];
extern const struct uapi_definition uverbs_def_obj_counters[];
extern const struct uapi_definition uverbs_def_obj_cq[];
extern const struct uapi_definition uverbs_def_obj_device[];
extern const struct uapi_definition uverbs_def_obj_dm[];
extern const struct uapi_definition uverbs_def_obj_flow_action[];
extern const struct uapi_definition uverbs_def_obj_intf[];
extern const struct uapi_definition uverbs_def_obj_mr[];
extern const struct uapi_definition uverbs_def_write_intf[];
static inline const struct uverbs_api_write_method *
uapi_get_method(const struct uverbs_api *uapi, u32 command)
{
u32 cmd_idx = command & IB_USER_VERBS_CMD_COMMAND_MASK;
if (command & ~(u32)(IB_USER_VERBS_CMD_FLAG_EXTENDED |
IB_USER_VERBS_CMD_COMMAND_MASK))
return ERR_PTR(-EINVAL);
if (command & IB_USER_VERBS_CMD_FLAG_EXTENDED) {
if (cmd_idx >= uapi->num_write_ex)
return ERR_PTR(-EOPNOTSUPP);
return uapi->write_ex_methods[cmd_idx];
}
if (cmd_idx >= uapi->num_write)
return ERR_PTR(-EOPNOTSUPP);
return uapi->write_methods[cmd_idx];
}
void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
struct ib_udata *udata, unsigned int attr_in,
unsigned int attr_out);
#endif /* RDMA_CORE_H */

205
sys/ofed/drivers/infiniband/core/uverbs.h
View File

@ -49,10 +49,15 @@
#include <linux/srcu.h>
#include <linux/rcupdate.h>
#include <linux/rbtree.h>
#include <linux/wait.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>
#include <rdma/uverbs_std_types.h>
#define UVERBS_MODULE_NAME ib_uverbs
#include <rdma/uverbs_named_ioctl.h>
static inline void
ib_uverbs_init_udata(struct ib_udata *udata,
@ -90,53 +95,76 @@ ib_uverbs_init_udata_buf_or_null(struct ib_udata *udata,
* an asynchronous event queue file is created and released when the
* event file is closed.
*
* struct ib_uverbs_event_file: One reference is held by the VFS and
* released when the file is closed. For asynchronous event files,
* another reference is held by the corresponding main context file
* and released when that file is closed. For completion event files,
* a reference is taken when a CQ is created that uses the file, and
* released when the CQ is destroyed.
* struct ib_uverbs_event_queue: Base structure for
* struct ib_uverbs_async_event_file and struct ib_uverbs_completion_event_file.
* One reference is held by the VFS and released when the file is closed.
* For asynchronous event files, another reference is held by the corresponding
* main context file and released when that file is closed. For completion
* event files, a reference is taken when a CQ is created that uses the file,
* and released when the CQ is destroyed.
*/
struct ib_uverbs_device {
atomic_t refcount;
int num_comp_vectors;
u32 num_comp_vectors;
struct completion comp;
struct device *dev;
struct device dev;
struct ib_device __rcu *ib_dev;
int devnum;
struct cdev cdev;
struct rb_root xrcd_tree;
struct mutex xrcd_tree_mutex;
struct kobject kobj;
struct srcu_struct disassociate_srcu;
struct mutex lists_mutex; /* protect lists */
struct list_head uverbs_file_list;
struct list_head uverbs_events_file_list;
struct uverbs_api *uapi;
};
struct ib_uverbs_event_file {
struct kref ref;
int is_async;
struct ib_uverbs_file *uverbs_file;
struct ib_uverbs_event_queue {
spinlock_t lock;
int is_closed;
wait_queue_head_t poll_wait;
struct fasync_struct *async_queue;
struct list_head event_list;
struct list_head list;
};
struct ib_uverbs_async_event_file {
struct ib_uobject uobj;
struct ib_uverbs_event_queue ev_queue;
struct ib_event_handler event_handler;
};
struct ib_uverbs_completion_event_file {
struct ib_uobject uobj;
struct ib_uverbs_event_queue ev_queue;
};
struct ib_uverbs_file {
struct kref ref;
struct mutex mutex;
struct mutex cleanup_mutex; /* protect cleanup */
struct ib_uverbs_device *device;
struct mutex ucontext_lock;
/*
* ucontext must be accessed via ib_uverbs_get_ucontext() or with
* ucontext_lock held
*/
struct ib_ucontext *ucontext;
struct ib_event_handler event_handler;
struct ib_uverbs_event_file *async_file;
struct ib_uverbs_async_event_file *async_file;
struct list_head list;
int is_closed;
/*
* To access the uobjects list hw_destroy_rwsem must be held for write
* OR hw_destroy_rwsem held for read AND uobjects_lock held.
* hw_destroy_rwsem should be called across any destruction of the HW
* object of an associated uobject.
*/
struct rw_semaphore hw_destroy_rwsem;
spinlock_t uobjects_lock;
struct list_head uobjects;
struct mutex umap_lock;
struct list_head umaps;
struct xarray idr;
};
struct ib_uverbs_event {
@ -157,6 +185,7 @@ struct ib_uverbs_mcast_entry {
struct ib_uevent_object {
struct ib_uobject uobject;
/* List member for ib_uverbs_async_event_file list */
struct list_head event_list;
u32 events_reported;
};
@ -184,51 +213,40 @@ struct ib_uwq_object {
};
struct ib_ucq_object {
struct ib_uobject uobject;
struct ib_uverbs_file *uverbs_file;
struct ib_uevent_object uevent;
struct list_head comp_list;
struct list_head async_list;
u32 comp_events_reported;
u32 async_events_reported;
};
extern spinlock_t ib_uverbs_idr_lock;
extern struct idr ib_uverbs_pd_idr;
extern struct idr ib_uverbs_mr_idr;
extern struct idr ib_uverbs_mw_idr;
extern struct idr ib_uverbs_ah_idr;
extern struct idr ib_uverbs_cq_idr;
extern struct idr ib_uverbs_qp_idr;
extern struct idr ib_uverbs_srq_idr;
extern struct idr ib_uverbs_xrcd_idr;
extern struct idr ib_uverbs_rule_idr;
extern struct idr ib_uverbs_wq_idr;
extern struct idr ib_uverbs_rwq_ind_tbl_idr;
extern const struct file_operations uverbs_event_fops;
extern const struct file_operations uverbs_async_event_fops;
void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue);
void ib_uverbs_init_async_event_file(struct ib_uverbs_async_event_file *ev_file);
void ib_uverbs_free_event_queue(struct ib_uverbs_event_queue *event_queue);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res);
void idr_remove_uobj(struct idr *idp, struct ib_uobject *uobj);
int ib_alloc_ucontext(struct uverbs_attr_bundle *attrs);
int ib_init_ucontext(struct uverbs_attr_bundle *attrs);
struct file *ib_uverbs_alloc_event_file(struct ib_uverbs_file *uverbs_file,
struct ib_device *ib_dev,
int is_async);
void ib_uverbs_free_async_event_file(struct ib_uverbs_file *uverbs_file);
struct ib_uverbs_event_file *ib_uverbs_lookup_comp_file(int fd);
void ib_uverbs_release_ucq(struct ib_uverbs_file *file,
struct ib_uverbs_event_file *ev_file,
void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,
struct ib_ucq_object *uobj);
void ib_uverbs_release_uevent(struct ib_uverbs_file *file,
struct ib_uevent_object *uobj);
void ib_uverbs_release_uevent(struct ib_uevent_object *uobj);
void ib_uverbs_release_file(struct kref *ref);
void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context);
void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr);
void ib_uverbs_event_handler(struct ib_event_handler *handler,
struct ib_event *event);
void ib_uverbs_dealloc_xrcd(struct ib_uverbs_device *dev, struct ib_xrcd *xrcd);
int ib_uverbs_dealloc_xrcd(struct ib_uobject *uobject, struct ib_xrcd *xrcd,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs);
int uverbs_dealloc_mw(struct ib_mw *mw);
void ib_uverbs_detach_umcast(struct ib_qp *qp,
struct ib_uqp_object *uobj);
long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg);
struct ib_uverbs_flow_spec {
union {
@ -242,68 +260,45 @@ struct ib_uverbs_flow_spec {
};
struct ib_uverbs_flow_spec_eth eth;
struct ib_uverbs_flow_spec_ipv4 ipv4;
struct ib_uverbs_flow_spec_esp esp;
struct ib_uverbs_flow_spec_tcp_udp tcp_udp;
struct ib_uverbs_flow_spec_ipv6 ipv6;
struct ib_uverbs_flow_spec_action_tag flow_tag;
struct ib_uverbs_flow_spec_action_drop drop;
struct ib_uverbs_flow_spec_action_handle action;
struct ib_uverbs_flow_spec_action_count flow_count;
};
};
#define IB_UVERBS_DECLARE_CMD(name) \
ssize_t ib_uverbs_##name(struct ib_uverbs_file *file, \
struct ib_device *ib_dev, \
const char __user *buf, int in_len, \
int out_len)
int ib_uverbs_kern_spec_to_ib_spec_filter(enum ib_flow_spec_type type,
const void *kern_spec_mask,
const void *kern_spec_val,
size_t kern_filter_sz,
union ib_flow_spec *ib_spec);
IB_UVERBS_DECLARE_CMD(get_context);
IB_UVERBS_DECLARE_CMD(query_device);
IB_UVERBS_DECLARE_CMD(query_port);
IB_UVERBS_DECLARE_CMD(alloc_pd);
IB_UVERBS_DECLARE_CMD(dealloc_pd);
IB_UVERBS_DECLARE_CMD(reg_mr);
IB_UVERBS_DECLARE_CMD(rereg_mr);
IB_UVERBS_DECLARE_CMD(dereg_mr);
IB_UVERBS_DECLARE_CMD(alloc_mw);
IB_UVERBS_DECLARE_CMD(dealloc_mw);
IB_UVERBS_DECLARE_CMD(create_comp_channel);
IB_UVERBS_DECLARE_CMD(create_cq);
IB_UVERBS_DECLARE_CMD(resize_cq);
IB_UVERBS_DECLARE_CMD(poll_cq);
IB_UVERBS_DECLARE_CMD(req_notify_cq);
IB_UVERBS_DECLARE_CMD(destroy_cq);
IB_UVERBS_DECLARE_CMD(create_qp);
IB_UVERBS_DECLARE_CMD(open_qp);
IB_UVERBS_DECLARE_CMD(query_qp);
IB_UVERBS_DECLARE_CMD(modify_qp);
IB_UVERBS_DECLARE_CMD(destroy_qp);
IB_UVERBS_DECLARE_CMD(post_send);
IB_UVERBS_DECLARE_CMD(post_recv);
IB_UVERBS_DECLARE_CMD(post_srq_recv);
IB_UVERBS_DECLARE_CMD(create_ah);
IB_UVERBS_DECLARE_CMD(destroy_ah);
IB_UVERBS_DECLARE_CMD(attach_mcast);
IB_UVERBS_DECLARE_CMD(detach_mcast);
IB_UVERBS_DECLARE_CMD(create_srq);
IB_UVERBS_DECLARE_CMD(modify_srq);
IB_UVERBS_DECLARE_CMD(query_srq);
IB_UVERBS_DECLARE_CMD(destroy_srq);
IB_UVERBS_DECLARE_CMD(create_xsrq);
IB_UVERBS_DECLARE_CMD(open_xrcd);
IB_UVERBS_DECLARE_CMD(close_xrcd);
/*
* ib_uverbs_query_port_resp.port_cap_flags started out as just a copy of the
* PortInfo CapabilityMask, but was extended with unique bits.
*/
static inline u32 make_port_cap_flags(const struct ib_port_attr *attr)
{
u32 res;
#define IB_UVERBS_DECLARE_EX_CMD(name) \
int ib_uverbs_ex_##name(struct ib_uverbs_file *file, \
struct ib_device *ib_dev, \
struct ib_udata *ucore, \
struct ib_udata *uhw)
/* All IBA CapabilityMask bits are passed through here, except bit 26,
* which is overridden with IP_BASED_GIDS. This is due to a historical
* mistake in the implementation of IP_BASED_GIDS. Otherwise all other
* bits match the IBA definition across all kernel versions.
*/
res = attr->port_cap_flags & ~(u32)IB_UVERBS_PCF_IP_BASED_GIDS;
IB_UVERBS_DECLARE_EX_CMD(create_flow);
IB_UVERBS_DECLARE_EX_CMD(destroy_flow);
IB_UVERBS_DECLARE_EX_CMD(query_device);
IB_UVERBS_DECLARE_EX_CMD(create_cq);
IB_UVERBS_DECLARE_EX_CMD(create_qp);
IB_UVERBS_DECLARE_EX_CMD(create_wq);
IB_UVERBS_DECLARE_EX_CMD(modify_wq);
IB_UVERBS_DECLARE_EX_CMD(destroy_wq);
IB_UVERBS_DECLARE_EX_CMD(create_rwq_ind_table);
IB_UVERBS_DECLARE_EX_CMD(destroy_rwq_ind_table);
if (attr->ip_gids)
res |= IB_UVERBS_PCF_IP_BASED_GIDS;
return res;
}
void copy_port_attr_to_resp(struct ib_port_attr *attr,
struct ib_uverbs_query_port_resp *resp,
struct ib_device *ib_dev, u8 port_num);
#endif /* UVERBS_H */

4
sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_ib.c
View File

@ -70,7 +70,7 @@ struct ipoib_ah *ipoib_create_ah(struct ipoib_dev_priv *priv,
ah->last_send = 0;
kref_init(&ah->ref);
ah->ah = ib_create_ah(pd, attr);
ah->ah = ib_create_ah(pd, attr, RDMA_CREATE_AH_SLEEPABLE);
if (IS_ERR(ah->ah)) {
kfree(ah);
ah = NULL;
@ -572,7 +572,7 @@ static void __ipoib_reap_ah(struct ipoib_dev_priv *priv)
list_for_each_entry_safe(ah, tah, &priv->dead_ahs, list)
if ((int) priv->tx_tail - (int) ah->last_send >= 0) {
list_del(&ah->list);
ib_destroy_ah(ah->ah);
ib_destroy_ah(ah->ah, 0);
kfree(ah);
}

8
sys/ofed/drivers/infiniband/ulp/sdp/sdp_rx.c
View File

@ -750,12 +750,8 @@ sdp_rx_ring_destroy(struct sdp_sock *ssk)
}
if (ssk->rx_ring.cq) {
if (ib_destroy_cq(ssk->rx_ring.cq)) {
sdp_warn(ssk->socket, "destroy cq(%p) failed\n",
ssk->rx_ring.cq);
} else {
ssk->rx_ring.cq = NULL;
}
ib_destroy_cq(ssk->rx_ring.cq);
ssk->rx_ring.cq = NULL;
}
WARN_ON(ring_head(ssk->rx_ring) != ring_tail(ssk->rx_ring));

8
sys/ofed/drivers/infiniband/ulp/sdp/sdp_tx.c
View File

@ -476,12 +476,8 @@ sdp_tx_ring_destroy(struct sdp_sock *ssk)
}
if (ssk->tx_ring.cq) {
if (ib_destroy_cq(ssk->tx_ring.cq)) {
sdp_warn(ssk->socket, "destroy cq(%p) failed\n",
ssk->tx_ring.cq);
} else {
ssk->tx_ring.cq = NULL;
}
ib_destroy_cq(ssk->tx_ring.cq);
ssk->tx_ring.cq = NULL;
}
WARN_ON(ring_head(ssk->tx_ring) != ring_tail(ssk->tx_ring));

959
sys/ofed/include/rdma/ib_verbs.h
View File

File diff suppressed because it is too large Load Diff

124
sys/ofed/include/rdma/signature.h Normal file
View File

@ -0,0 +1,124 @@
/* SPDX-License-Identifier: (GPL-2.0 OR Linux-OpenIB) */
/*
* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved.
*/
#ifndef _RDMA_SIGNATURE_H_
#define _RDMA_SIGNATURE_H_
#include <linux/types.h>
enum ib_signature_prot_cap {
IB_PROT_T10DIF_TYPE_1 = 1,
IB_PROT_T10DIF_TYPE_2 = 1 << 1,
IB_PROT_T10DIF_TYPE_3 = 1 << 2,
};
enum ib_signature_guard_cap {
IB_GUARD_T10DIF_CRC = 1,
IB_GUARD_T10DIF_CSUM = 1 << 1,
};
/**
* enum ib_signature_type - Signature types
* @IB_SIG_TYPE_NONE: Unprotected.
* @IB_SIG_TYPE_T10_DIF: Type T10-DIF
*/
enum ib_signature_type {
IB_SIG_TYPE_NONE,
IB_SIG_TYPE_T10_DIF,
};
/**
* enum ib_t10_dif_bg_type - Signature T10-DIF block-guard types
* @IB_T10DIF_CRC: Corresponds to T10-PI mandated CRC checksum rules.
* @IB_T10DIF_CSUM: Corresponds to IP checksum rules.
*/
enum ib_t10_dif_bg_type {
IB_T10DIF_CRC,
IB_T10DIF_CSUM,
};
/**
* struct ib_t10_dif_domain - Parameters specific for T10-DIF
* domain.
* @bg_type: T10-DIF block guard type (CRC|CSUM)
* @pi_interval: protection information interval.
* @bg: seed of guard computation.
* @app_tag: application tag of guard block
* @ref_tag: initial guard block reference tag.
* @ref_remap: Indicate wethear the reftag increments each block
* @app_escape: Indicate to skip block check if apptag=0xffff
* @ref_escape: Indicate to skip block check if reftag=0xffffffff
* @apptag_check_mask: check bitmask of application tag.
*/
struct ib_t10_dif_domain {
enum ib_t10_dif_bg_type bg_type;
u16 pi_interval;
u16 bg;
u16 app_tag;
u32 ref_tag;
bool ref_remap;
bool app_escape;
bool ref_escape;
u16 apptag_check_mask;
};
/**
* struct ib_sig_domain - Parameters for signature domain
* @sig_type: specific signauture type
* @sig: union of all signature domain attributes that may
* be used to set domain layout.
*/
struct ib_sig_domain {
enum ib_signature_type sig_type;
union {
struct ib_t10_dif_domain dif;
} sig;
};
/**
* struct ib_sig_attrs - Parameters for signature handover operation
* @check_mask: bitmask for signature byte check (8 bytes)
* @mem: memory domain layout descriptor.
* @wire: wire domain layout descriptor.
* @meta_length: metadata length
*/
struct ib_sig_attrs {
u8 check_mask;
struct ib_sig_domain mem;
struct ib_sig_domain wire;
int meta_length;
};
enum ib_sig_err_type {
IB_SIG_BAD_GUARD,
IB_SIG_BAD_REFTAG,
IB_SIG_BAD_APPTAG,
};
/*
* Signature check masks (8 bytes in total) according to the T10-PI standard:
* -------- -------- ------------
* | GUARD | APPTAG | REFTAG |
* | 2B | 2B | 4B |
* -------- -------- ------------
*/
enum {
IB_SIG_CHECK_GUARD = 0xc0,
IB_SIG_CHECK_APPTAG = 0x30,
IB_SIG_CHECK_REFTAG = 0x0f,
};
/*
* struct ib_sig_err - signature error descriptor
*/
struct ib_sig_err {
enum ib_sig_err_type err_type;
u32 expected;
u32 actual;
u64 sig_err_offset;
u32 key;
};
#endif /* _RDMA_SIGNATURE_H_ */

958
sys/ofed/include/rdma/uverbs_ioctl.h Normal file
View File

@ -0,0 +1,958 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _UVERBS_IOCTL_
#define _UVERBS_IOCTL_
#include <rdma/uverbs_types.h>
#include <linux/uaccess.h>
#include <rdma/rdma_user_ioctl.h>
#include <rdma/ib_user_ioctl_verbs.h>
#include <rdma/ib_user_ioctl_cmds.h>
/*
* =======================================
* Verbs action specifications
* =======================================
*/
enum uverbs_attr_type {
UVERBS_ATTR_TYPE_NA,
UVERBS_ATTR_TYPE_PTR_IN,
UVERBS_ATTR_TYPE_PTR_OUT,
UVERBS_ATTR_TYPE_IDR,
UVERBS_ATTR_TYPE_FD,
UVERBS_ATTR_TYPE_ENUM_IN,
UVERBS_ATTR_TYPE_IDRS_ARRAY,
};
enum uverbs_obj_access {
UVERBS_ACCESS_READ,
UVERBS_ACCESS_WRITE,
UVERBS_ACCESS_NEW,
UVERBS_ACCESS_DESTROY
};
/* Specification of a single attribute inside the ioctl message */
/* good size 16 */
struct uverbs_attr_spec {
u8 type;
/*
* Support extending attributes by length. Allow the user to provide
* more bytes than ptr.len, but check that everything after is zero'd
* by the user.
*/
u8 zero_trailing:1;
/*
* Valid only for PTR_IN. Allocate and copy the data inside
* the parser
*/
u8 alloc_and_copy:1;
u8 mandatory:1;
/* True if this is from UVERBS_ATTR_UHW */
u8 is_udata:1;
union {
struct {
/* Current known size to kernel */
u16 len;
/* User isn't allowed to provide something < min_len */
u16 min_len;
} ptr;
struct {
/*
* higher bits mean the namespace and lower bits mean
* the type id within the namespace.
*/
u16 obj_type;
u8 access;
} obj;
struct {
u8 num_elems;
} enum_def;
} u;
/* This weird split lets us remove some padding */
union {
struct {
/*
* The enum attribute can select one of the attributes
* contained in the ids array. Currently only PTR_IN
* attributes are supported in the ids array.
*/
const struct uverbs_attr_spec *ids;
} enum_def;
struct {
/*
* higher bits mean the namespace and lower bits mean
* the type id within the namespace.
*/
u16 obj_type;
u16 min_len;
u16 max_len;
u8 access;
} objs_arr;
} u2;
};
/*
* Information about the API is loaded into a radix tree. For IOCTL we start
* with a tuple of:
* object_id, attr_id, method_id
*
* Which is a 48 bit value, with most of the bits guaranteed to be zero. Based
* on the current kernel support this is compressed into 16 bit key for the
* radix tree. Since this compression is entirely internal to the kernel the
* below limits can be revised if the kernel gains additional data.
*
* With 64 leafs per node this is a 3 level radix tree.
*
* The tree encodes multiple types, and uses a scheme where OBJ_ID,0,0 returns
* the object slot, and OBJ_ID,METH_ID,0 and returns the method slot.
*
* This also encodes the tables for the write() and write() extended commands
* using the coding
* OBJ_ID,UVERBS_API_METHOD_IS_WRITE,command #
* OBJ_ID,UVERBS_API_METHOD_IS_WRITE_EX,command_ex #
* ie the WRITE path is treated as a special method type in the ioctl
* framework.
*/
enum uapi_radix_data {
UVERBS_API_NS_FLAG = 1U << UVERBS_ID_NS_SHIFT,
UVERBS_API_ATTR_KEY_BITS = 6,
UVERBS_API_ATTR_KEY_MASK = GENMASK(UVERBS_API_ATTR_KEY_BITS - 1, 0),
UVERBS_API_ATTR_BKEY_LEN = (1 << UVERBS_API_ATTR_KEY_BITS) - 1,
UVERBS_API_WRITE_KEY_NUM = 1 << UVERBS_API_ATTR_KEY_BITS,
UVERBS_API_METHOD_KEY_BITS = 5,
UVERBS_API_METHOD_KEY_SHIFT = UVERBS_API_ATTR_KEY_BITS,
UVERBS_API_METHOD_KEY_NUM_CORE = 22,
UVERBS_API_METHOD_IS_WRITE = 30 << UVERBS_API_METHOD_KEY_SHIFT,
UVERBS_API_METHOD_IS_WRITE_EX = 31 << UVERBS_API_METHOD_KEY_SHIFT,
UVERBS_API_METHOD_KEY_NUM_DRIVER =
(UVERBS_API_METHOD_IS_WRITE >> UVERBS_API_METHOD_KEY_SHIFT) -
UVERBS_API_METHOD_KEY_NUM_CORE,
UVERBS_API_METHOD_KEY_MASK = GENMASK(
UVERBS_API_METHOD_KEY_BITS + UVERBS_API_METHOD_KEY_SHIFT - 1,
UVERBS_API_METHOD_KEY_SHIFT),
UVERBS_API_OBJ_KEY_BITS = 5,
UVERBS_API_OBJ_KEY_SHIFT =
UVERBS_API_METHOD_KEY_BITS + UVERBS_API_METHOD_KEY_SHIFT,
UVERBS_API_OBJ_KEY_NUM_CORE = 20,
UVERBS_API_OBJ_KEY_NUM_DRIVER =
(1 << UVERBS_API_OBJ_KEY_BITS) - UVERBS_API_OBJ_KEY_NUM_CORE,
UVERBS_API_OBJ_KEY_MASK = GENMASK(31, UVERBS_API_OBJ_KEY_SHIFT),
/* This id guaranteed to not exist in the radix tree */
UVERBS_API_KEY_ERR = 0xFFFFFFFF,
};
static inline __attribute_const__ u32 uapi_key_obj(u32 id)
{
if (id & UVERBS_API_NS_FLAG) {
id &= ~UVERBS_API_NS_FLAG;
if (id >= UVERBS_API_OBJ_KEY_NUM_DRIVER)
return UVERBS_API_KEY_ERR;
id = id + UVERBS_API_OBJ_KEY_NUM_CORE;
} else {
if (id >= UVERBS_API_OBJ_KEY_NUM_CORE)
return UVERBS_API_KEY_ERR;
}
return id << UVERBS_API_OBJ_KEY_SHIFT;
}
static inline __attribute_const__ bool uapi_key_is_object(u32 key)
{
return (key & ~UVERBS_API_OBJ_KEY_MASK) == 0;
}
static inline __attribute_const__ u32 uapi_key_ioctl_method(u32 id)
{
if (id & UVERBS_API_NS_FLAG) {
id &= ~UVERBS_API_NS_FLAG;
if (id >= UVERBS_API_METHOD_KEY_NUM_DRIVER)
return UVERBS_API_KEY_ERR;
id = id + UVERBS_API_METHOD_KEY_NUM_CORE;
} else {
id++;
if (id >= UVERBS_API_METHOD_KEY_NUM_CORE)
return UVERBS_API_KEY_ERR;
}
return id << UVERBS_API_METHOD_KEY_SHIFT;
}
static inline __attribute_const__ u32 uapi_key_write_method(u32 id)
{
if (id >= UVERBS_API_WRITE_KEY_NUM)
return UVERBS_API_KEY_ERR;
return UVERBS_API_METHOD_IS_WRITE | id;
}
static inline __attribute_const__ u32 uapi_key_write_ex_method(u32 id)
{
if (id >= UVERBS_API_WRITE_KEY_NUM)
return UVERBS_API_KEY_ERR;
return UVERBS_API_METHOD_IS_WRITE_EX | id;
}
static inline __attribute_const__ u32
uapi_key_attr_to_ioctl_method(u32 attr_key)
{
return attr_key &
(UVERBS_API_OBJ_KEY_MASK | UVERBS_API_METHOD_KEY_MASK);
}
static inline __attribute_const__ bool uapi_key_is_ioctl_method(u32 key)
{
unsigned int method = key & UVERBS_API_METHOD_KEY_MASK;
return method != 0 && method < UVERBS_API_METHOD_IS_WRITE &&
(key & UVERBS_API_ATTR_KEY_MASK) == 0;
}
static inline __attribute_const__ bool uapi_key_is_write_method(u32 key)
{
return (key & UVERBS_API_METHOD_KEY_MASK) == UVERBS_API_METHOD_IS_WRITE;
}
static inline __attribute_const__ bool uapi_key_is_write_ex_method(u32 key)
{
return (key & UVERBS_API_METHOD_KEY_MASK) ==
UVERBS_API_METHOD_IS_WRITE_EX;
}
static inline __attribute_const__ u32 uapi_key_attrs_start(u32 ioctl_method_key)
{
/* 0 is the method slot itself */
return ioctl_method_key + 1;
}
static inline __attribute_const__ u32 uapi_key_attr(u32 id)
{
/*
* The attr is designed to fit in the typical single radix tree node
* of 64 entries. Since allmost all methods have driver attributes we
* organize things so that the driver and core attributes interleave to
* reduce the length of the attributes array in typical cases.
*/
if (id & UVERBS_API_NS_FLAG) {
id &= ~UVERBS_API_NS_FLAG;
id++;
if (id >= 1 << (UVERBS_API_ATTR_KEY_BITS - 1))
return UVERBS_API_KEY_ERR;
id = (id << 1) | 0;
} else {
if (id >= 1 << (UVERBS_API_ATTR_KEY_BITS - 1))
return UVERBS_API_KEY_ERR;
id = (id << 1) | 1;
}
return id;
}
/* Only true for ioctl methods */
static inline __attribute_const__ bool uapi_key_is_attr(u32 key)
{
unsigned int method = key & UVERBS_API_METHOD_KEY_MASK;
return method != 0 && method < UVERBS_API_METHOD_IS_WRITE &&
(key & UVERBS_API_ATTR_KEY_MASK) != 0;
}
/*
* This returns a value in the range [0 to UVERBS_API_ATTR_BKEY_LEN),
* basically it undoes the reservation of 0 in the ID numbering. attr_key
* must already be masked with UVERBS_API_ATTR_KEY_MASK, or be the output of
* uapi_key_attr().
*/
static inline __attribute_const__ u32 uapi_bkey_attr(u32 attr_key)
{
return attr_key - 1;
}
static inline __attribute_const__ u32 uapi_bkey_to_key_attr(u32 attr_bkey)
{
return attr_bkey + 1;
}
/*
* =======================================
* Verbs definitions
* =======================================
*/
struct uverbs_attr_def {
u16 id;
struct uverbs_attr_spec attr;
};
struct uverbs_method_def {
u16 id;
/* Combination of bits from enum UVERBS_ACTION_FLAG_XXXX */
u32 flags;
size_t num_attrs;
const struct uverbs_attr_def * const (*attrs)[];
int (*handler)(struct uverbs_attr_bundle *attrs);
};
struct uverbs_object_def {
u16 id;
const struct uverbs_obj_type *type_attrs;
size_t num_methods;
const struct uverbs_method_def * const (*methods)[];
};
enum uapi_definition_kind {
UAPI_DEF_END = 0,
UAPI_DEF_OBJECT_START,
UAPI_DEF_WRITE,
UAPI_DEF_CHAIN_OBJ_TREE,
UAPI_DEF_CHAIN,
UAPI_DEF_IS_SUPPORTED_FUNC,
UAPI_DEF_IS_SUPPORTED_DEV_FN,
};
enum uapi_definition_scope {
UAPI_SCOPE_OBJECT = 1,
UAPI_SCOPE_METHOD = 2,
};
struct uapi_definition {
u8 kind;
u8 scope;
union {
struct {
u16 object_id;
} object_start;
struct {
u16 command_num;
u8 is_ex:1;
u8 has_udata:1;
u8 has_resp:1;
u8 req_size;
u8 resp_size;
} write;
};
union {
bool (*func_is_supported)(struct ib_device *device);
int (*func_write)(struct uverbs_attr_bundle *attrs);
const struct uapi_definition *chain;
const struct uverbs_object_def *chain_obj_tree;
size_t needs_fn_offset;
};
};
/* Define things connected to object_id */
#define DECLARE_UVERBS_OBJECT(_object_id, ...) \
{ \
.kind = UAPI_DEF_OBJECT_START, \
.object_start = { .object_id = _object_id }, \
}, \
##__VA_ARGS__
/* Use in a var_args of DECLARE_UVERBS_OBJECT */
#define DECLARE_UVERBS_WRITE(_command_num, _func, _cmd_desc, ...) \
{ \
.kind = UAPI_DEF_WRITE, \
.scope = UAPI_SCOPE_OBJECT, \
.write = { .is_ex = 0, .command_num = _command_num }, \
.func_write = _func, \
_cmd_desc, \
}, \
##__VA_ARGS__
/* Use in a var_args of DECLARE_UVERBS_OBJECT */
#define DECLARE_UVERBS_WRITE_EX(_command_num, _func, _cmd_desc, ...) \
{ \
.kind = UAPI_DEF_WRITE, \
.scope = UAPI_SCOPE_OBJECT, \
.write = { .is_ex = 1, .command_num = _command_num }, \
.func_write = _func, \
_cmd_desc, \
}, \
##__VA_ARGS__
/*
* Object is only supported if the function pointer named ibdev_fn in struct
* ib_device is not NULL.
*/
#define UAPI_DEF_OBJ_NEEDS_FN(ibdev_fn) \
{ \
.kind = UAPI_DEF_IS_SUPPORTED_DEV_FN, \
.scope = UAPI_SCOPE_OBJECT, \
.needs_fn_offset = \
offsetof(struct ib_device, ibdev_fn) + \
BUILD_BUG_ON_ZERO( \
sizeof(((struct ib_device *)0)->ibdev_fn) != \
sizeof(void *)), \
}
/*
* Method is only supported if the function pointer named ibdev_fn in struct
* ib_device is not NULL.
*/
#define UAPI_DEF_METHOD_NEEDS_FN(ibdev_fn) \
{ \
.kind = UAPI_DEF_IS_SUPPORTED_DEV_FN, \
.scope = UAPI_SCOPE_METHOD, \
.needs_fn_offset = \
offsetof(struct ib_device, ibdev_fn) + \
BUILD_BUG_ON_ZERO( \
sizeof(((struct ib_device *)0)->ibdev_fn) != \
sizeof(void *)), \
}
/* Call a function to determine if the entire object is supported or not */
#define UAPI_DEF_IS_OBJ_SUPPORTED(_func) \
{ \
.kind = UAPI_DEF_IS_SUPPORTED_FUNC, \
.scope = UAPI_SCOPE_OBJECT, .func_is_supported = _func, \
}
/* Include another struct uapi_definition in this one */
#define UAPI_DEF_CHAIN(_def_var) \
{ \
.kind = UAPI_DEF_CHAIN, .chain = _def_var, \
}
/* Temporary until the tree base description is replaced */
#define UAPI_DEF_CHAIN_OBJ_TREE(_object_enum, _object_ptr, ...) \
{ \
.kind = UAPI_DEF_CHAIN_OBJ_TREE, \
.object_start = { .object_id = _object_enum }, \
.chain_obj_tree = _object_ptr, \
}, \
##__VA_ARGS__
#define UAPI_DEF_CHAIN_OBJ_TREE_NAMED(_object_enum, ...) \
UAPI_DEF_CHAIN_OBJ_TREE(_object_enum, &UVERBS_OBJECT(_object_enum), \
##__VA_ARGS__)
/*
* =======================================
* Attribute Specifications
* =======================================
*/
#define UVERBS_ATTR_SIZE(_min_len, _len) \
.u.ptr.min_len = _min_len, .u.ptr.len = _len
#define UVERBS_ATTR_NO_DATA() UVERBS_ATTR_SIZE(0, 0)
/*
* Specifies a uapi structure that cannot be extended. The user must always
* supply the whole structure and nothing more. The structure must be declared
* in a header under include/uapi/rdma.
*/
#define UVERBS_ATTR_TYPE(_type) \
.u.ptr.min_len = sizeof(_type), .u.ptr.len = sizeof(_type)
/*
* Specifies a uapi structure where the user must provide at least up to
* member 'last'. Anything after last and up until the end of the structure
* can be non-zero, anything longer than the end of the structure must be
* zero. The structure must be declared in a header under include/uapi/rdma.
*/
#define UVERBS_ATTR_STRUCT(_type, _last) \
.zero_trailing = 1, \
UVERBS_ATTR_SIZE(((uintptr_t)(&((_type *)0)->_last + 1)), \
sizeof(_type))
/*
* Specifies at least min_len bytes must be passed in, but the amount can be
* larger, up to the protocol maximum size. No check for zeroing is done.
*/
#define UVERBS_ATTR_MIN_SIZE(_min_len) UVERBS_ATTR_SIZE(_min_len, USHRT_MAX)
/* Must be used in the '...' of any UVERBS_ATTR */
#define UA_ALLOC_AND_COPY .alloc_and_copy = 1
#define UA_MANDATORY .mandatory = 1
#define UA_OPTIONAL .mandatory = 0
/*
* min_len must be bigger than 0 and _max_len must be smaller than 4095. Only
* READ\WRITE accesses are supported.
*/
#define UVERBS_ATTR_IDRS_ARR(_attr_id, _idr_type, _access, _min_len, _max_len, \
...) \
(&(const struct uverbs_attr_def){ \
.id = (_attr_id) + \
BUILD_BUG_ON_ZERO((_min_len) == 0 || \
(_max_len) > \
PAGE_SIZE / sizeof(void *) || \
(_min_len) > (_max_len) || \
(_access) == UVERBS_ACCESS_NEW || \
(_access) == UVERBS_ACCESS_DESTROY), \
.attr = { .type = UVERBS_ATTR_TYPE_IDRS_ARRAY, \
.u2.objs_arr.obj_type = _idr_type, \
.u2.objs_arr.access = _access, \
.u2.objs_arr.min_len = _min_len, \
.u2.objs_arr.max_len = _max_len, \
__VA_ARGS__ } })
/*
* Only for use with UVERBS_ATTR_IDR, allows any uobject type to be accepted,
* the user must validate the type of the uobject instead.
*/
#define UVERBS_IDR_ANY_OBJECT 0xFFFF
#define UVERBS_ATTR_IDR(_attr_id, _idr_type, _access, ...) \
(&(const struct uverbs_attr_def){ \
.id = _attr_id, \
.attr = { .type = UVERBS_ATTR_TYPE_IDR, \
.u.obj.obj_type = _idr_type, \
.u.obj.access = _access, \
__VA_ARGS__ } })
#define UVERBS_ATTR_FD(_attr_id, _fd_type, _access, ...) \
(&(const struct uverbs_attr_def){ \
.id = (_attr_id) + \
BUILD_BUG_ON_ZERO((_access) != UVERBS_ACCESS_NEW && \
(_access) != UVERBS_ACCESS_READ), \
.attr = { .type = UVERBS_ATTR_TYPE_FD, \
.u.obj.obj_type = _fd_type, \
.u.obj.access = _access, \
__VA_ARGS__ } })
#define UVERBS_ATTR_PTR_IN(_attr_id, _type, ...) \
(&(const struct uverbs_attr_def){ \
.id = _attr_id, \
.attr = { .type = UVERBS_ATTR_TYPE_PTR_IN, \
_type, \
__VA_ARGS__ } })
#define UVERBS_ATTR_PTR_OUT(_attr_id, _type, ...) \
(&(const struct uverbs_attr_def){ \
.id = _attr_id, \
.attr = { .type = UVERBS_ATTR_TYPE_PTR_OUT, \
_type, \
__VA_ARGS__ } })
/* _enum_arry should be a 'static const union uverbs_attr_spec[]' */
#define UVERBS_ATTR_ENUM_IN(_attr_id, _enum_arr, ...) \
(&(const struct uverbs_attr_def){ \
.id = _attr_id, \
.attr = { .type = UVERBS_ATTR_TYPE_ENUM_IN, \
.u2.enum_def.ids = _enum_arr, \
.u.enum_def.num_elems = ARRAY_SIZE(_enum_arr), \
__VA_ARGS__ }, \
})
/* An input value that is a member in the enum _enum_type. */
#define UVERBS_ATTR_CONST_IN(_attr_id, _enum_type, ...) \
UVERBS_ATTR_PTR_IN( \
_attr_id, \
UVERBS_ATTR_SIZE( \
sizeof(u64) + BUILD_BUG_ON_ZERO(!sizeof(_enum_type)), \
sizeof(u64)), \
__VA_ARGS__)
/*
* An input value that is a bitwise combination of values of _enum_type.
* This permits the flag value to be passed as either a u32 or u64, it must
* be retrieved via uverbs_get_flag().
*/
#define UVERBS_ATTR_FLAGS_IN(_attr_id, _enum_type, ...) \
UVERBS_ATTR_PTR_IN( \
_attr_id, \
UVERBS_ATTR_SIZE(sizeof(u32) + BUILD_BUG_ON_ZERO( \
!sizeof(_enum_type *)), \
sizeof(u64)), \
__VA_ARGS__)
/*
* This spec is used in order to pass information to the hardware driver in a
* legacy way. Every verb that could get driver specific data should get this
* spec.
*/
#define UVERBS_ATTR_UHW() \
UVERBS_ATTR_PTR_IN(UVERBS_ATTR_UHW_IN, \
UVERBS_ATTR_MIN_SIZE(0), \
UA_OPTIONAL, \
.is_udata = 1), \
UVERBS_ATTR_PTR_OUT(UVERBS_ATTR_UHW_OUT, \
UVERBS_ATTR_MIN_SIZE(0), \
UA_OPTIONAL, \
.is_udata = 1)
/* =================================================
* Parsing infrastructure
* =================================================
*/
struct uverbs_ptr_attr {
/*
* If UVERBS_ATTR_SPEC_F_ALLOC_AND_COPY is set then the 'ptr' is
* used.
*/
union {
void *ptr;
u64 data;
};
u16 len;
u16 uattr_idx;
u8 enum_id;
};
struct uverbs_obj_attr {
struct ib_uobject *uobject;
const struct uverbs_api_attr *attr_elm;
};
struct uverbs_objs_arr_attr {
struct ib_uobject **uobjects;
u16 len;
};
struct uverbs_attr {
union {
struct uverbs_ptr_attr ptr_attr;
struct uverbs_obj_attr obj_attr;
struct uverbs_objs_arr_attr objs_arr_attr;
};
};
struct uverbs_attr_bundle {
struct ib_udata driver_udata;
struct ib_udata ucore;
struct ib_uverbs_file *ufile;
struct ib_ucontext *context;
DECLARE_BITMAP(attr_present, UVERBS_API_ATTR_BKEY_LEN);
struct uverbs_attr attrs[0];
};
static inline bool uverbs_attr_is_valid(const struct uverbs_attr_bundle *attrs_bundle,
unsigned int idx)
{
return test_bit(uapi_bkey_attr(uapi_key_attr(idx)),
attrs_bundle->attr_present);
}
/**
* rdma_udata_to_drv_context - Helper macro to get the driver's context out of
* ib_udata which is embedded in uverbs_attr_bundle.
*
* If udata is not NULL this cannot fail. Otherwise a NULL udata will result
* in a NULL ucontext pointer, as a safety precaution. Callers should be using
* 'udata' to determine if the driver call is in user or kernel mode, not
* 'ucontext'.
*
*/
#define rdma_udata_to_drv_context(udata, drv_dev_struct, member) \
(udata ? container_of(container_of(udata, struct uverbs_attr_bundle, \
driver_udata) \
->context, \
drv_dev_struct, member) : \
(drv_dev_struct *)NULL)
#define IS_UVERBS_COPY_ERR(_ret) ((_ret) && (_ret) != -ENOENT)
static inline const struct uverbs_attr *uverbs_attr_get(const struct uverbs_attr_bundle *attrs_bundle,
u16 idx)
{
if (!uverbs_attr_is_valid(attrs_bundle, idx))
return ERR_PTR(-ENOENT);
return &attrs_bundle->attrs[uapi_bkey_attr(uapi_key_attr(idx))];
}
static inline int uverbs_attr_get_enum_id(const struct uverbs_attr_bundle *attrs_bundle,
u16 idx)
{
const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
return attr->ptr_attr.enum_id;
}
static inline void *uverbs_attr_get_obj(const struct uverbs_attr_bundle *attrs_bundle,
u16 idx)
{
const struct uverbs_attr *attr;
attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr))
return ERR_CAST(attr);
return attr->obj_attr.uobject->object;
}
static inline struct ib_uobject *uverbs_attr_get_uobject(const struct uverbs_attr_bundle *attrs_bundle,
u16 idx)
{
const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr))
return ERR_CAST(attr);
return attr->obj_attr.uobject;
}
static inline int
uverbs_attr_get_len(const struct uverbs_attr_bundle *attrs_bundle, u16 idx)
{
const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
return attr->ptr_attr.len;
}
/*
* uverbs_attr_ptr_get_array_size() - Get array size pointer by a ptr
* attribute.
* @attrs: The attribute bundle
* @idx: The ID of the attribute
* @elem_size: The size of the element in the array
*/
static inline int
uverbs_attr_ptr_get_array_size(struct uverbs_attr_bundle *attrs, u16 idx,
size_t elem_size)
{
int size = uverbs_attr_get_len(attrs, idx);
if (size < 0)
return size;
if (size % elem_size)
return -EINVAL;
return size / elem_size;
}
/**
* uverbs_attr_get_uobjs_arr() - Provides array's properties for attribute for
* UVERBS_ATTR_TYPE_IDRS_ARRAY.
* @arr: Returned pointer to array of pointers for uobjects or NULL if
* the attribute isn't provided.
*
* Return: The array length or 0 if no attribute was provided.
*/
static inline int uverbs_attr_get_uobjs_arr(
const struct uverbs_attr_bundle *attrs_bundle, u16 attr_idx,
struct ib_uobject ***arr)
{
const struct uverbs_attr *attr =
uverbs_attr_get(attrs_bundle, attr_idx);
if (IS_ERR(attr)) {
*arr = NULL;
return 0;
}
*arr = attr->objs_arr_attr.uobjects;
return attr->objs_arr_attr.len;
}
static inline bool uverbs_attr_ptr_is_inline(const struct uverbs_attr *attr)
{
return attr->ptr_attr.len <= sizeof(attr->ptr_attr.data);
}
static inline void *uverbs_attr_get_alloced_ptr(
const struct uverbs_attr_bundle *attrs_bundle, u16 idx)
{
struct uverbs_attr *attr = __DECONST(struct uverbs_attr *, uverbs_attr_get(attrs_bundle, idx));
if (IS_ERR(attr))
return (void *)attr;
return uverbs_attr_ptr_is_inline(attr) ?
(void *)&attr->ptr_attr.data : attr->ptr_attr.ptr;
}
static inline int _uverbs_copy_from(void *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx,
size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx);
if (IS_ERR(attr))
return PTR_ERR(attr);
/*
* Validation ensures attr->ptr_attr.len >= size. If the caller is
* using UVERBS_ATTR_SPEC_F_MIN_SZ_OR_ZERO then it must call
* uverbs_copy_from_or_zero.
*/
if (unlikely(size < attr->ptr_attr.len))
return -EINVAL;
if (uverbs_attr_ptr_is_inline(attr))
memcpy(to, &attr->ptr_attr.data, attr->ptr_attr.len);
else if (copy_from_user(to, u64_to_user_ptr(attr->ptr_attr.data),
attr->ptr_attr.len))
return -EFAULT;
return 0;
}
static inline int _uverbs_copy_from_or_zero(void *to,
const struct uverbs_attr_bundle *attrs_bundle,
size_t idx,
size_t size)
{
const struct uverbs_attr *attr = uverbs_attr_get(attrs_bundle, idx);
size_t min_size;
if (IS_ERR(attr))
return PTR_ERR(attr);
min_size = min_t(size_t, size, attr->ptr_attr.len);
if (uverbs_attr_ptr_is_inline(attr))
memcpy(to, &attr->ptr_attr.data, min_size);
else if (copy_from_user(to, u64_to_user_ptr(attr->ptr_attr.data),
min_size))
return -EFAULT;
if (size > min_size)
memset((char *)to + min_size, 0, size - min_size);
return 0;
}
#define uverbs_copy_from(to, attrs_bundle, idx) \
_uverbs_copy_from(to, attrs_bundle, idx, sizeof(*to))
#define uverbs_copy_from_or_zero(to, attrs_bundle, idx) \
_uverbs_copy_from_or_zero(to, attrs_bundle, idx, sizeof(*to))
static inline struct ib_ucontext *
ib_uverbs_get_ucontext(const struct uverbs_attr_bundle *attrs)
{
return ib_uverbs_get_ucontext_file(attrs->ufile);
}
#if 1 /* IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) */
int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits);
int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits);
int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle, size_t idx,
const void *from, size_t size);
__malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
gfp_t flags);
static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
size_t size)
{
return _uverbs_alloc(bundle, size, GFP_KERNEL);
}
static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle,
size_t size)
{
return _uverbs_alloc(bundle, size, GFP_KERNEL | __GFP_ZERO);
}
int _uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, s64 lower_bound, u64 upper_bound,
s64 *def_val);
int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
size_t idx, const void *from, size_t size);
#else
static inline int
uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
return -EINVAL;
}
static inline int
uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, u64 allowed_bits)
{
return -EINVAL;
}
static inline int uverbs_copy_to(const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, const void *from, size_t size)
{
return -EINVAL;
}
static inline __malloc void *uverbs_alloc(struct uverbs_attr_bundle *bundle,
size_t size)
{
return ERR_PTR(-EINVAL);
}
static inline __malloc void *uverbs_zalloc(struct uverbs_attr_bundle *bundle,
size_t size)
{
return ERR_PTR(-EINVAL);
}
static inline int
_uverbs_get_const(s64 *to, const struct uverbs_attr_bundle *attrs_bundle,
size_t idx, s64 lower_bound, u64 upper_bound,
s64 *def_val)
{
return -EINVAL;
}
static inline int
uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
size_t idx, const void *from, size_t size)
{
return -EINVAL;
}
#endif
#define uverbs_get_const(_to, _attrs_bundle, _idx) \
({ \
s64 _val; \
int _ret = _uverbs_get_const(&_val, _attrs_bundle, _idx, \
type_min(typeof(*_to)), \
type_max(typeof(*_to)), NULL); \
(*_to) = _val; \
_ret; \
})
#define uverbs_get_const_default(_to, _attrs_bundle, _idx, _default) \
({ \
s64 _val; \
s64 _def_val = _default; \
int _ret = \
_uverbs_get_const(&_val, _attrs_bundle, _idx, \
type_min(typeof(*_to)), \
type_max(typeof(*_to)), &_def_val); \
(*_to) = _val; \
_ret; \
})
#endif

124
sys/ofed/include/rdma/uverbs_named_ioctl.h Normal file
View File

@ -0,0 +1,124 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _UVERBS_NAMED_IOCTL_
#define _UVERBS_NAMED_IOCTL_
#include <rdma/uverbs_ioctl.h>
#ifndef UVERBS_MODULE_NAME
#error "Please #define UVERBS_MODULE_NAME before including rdma/uverbs_named_ioctl.h"
#endif
#define _UVERBS_PASTE(x, y) x ## y
#define _UVERBS_NAME(x, y) _UVERBS_PASTE(x, y)
#define UVERBS_METHOD(id) _UVERBS_NAME(UVERBS_MODULE_NAME, _method_##id)
#define UVERBS_HANDLER(id) _UVERBS_NAME(UVERBS_MODULE_NAME, _handler_##id)
#define UVERBS_OBJECT(id) _UVERBS_NAME(UVERBS_MODULE_NAME, _object_##id)
/* These are static so they do not need to be qualified */
#define UVERBS_METHOD_ATTRS(method_id) _method_attrs_##method_id
#define UVERBS_OBJECT_METHODS(object_id) _object_methods_##object_id
#define DECLARE_UVERBS_NAMED_METHOD(_method_id, ...) \
static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \
_method_id)[] = { __VA_ARGS__ }; \
static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \
.id = _method_id, \
.handler = UVERBS_HANDLER(_method_id), \
.num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \
.attrs = &UVERBS_METHOD_ATTRS(_method_id), \
}
/* Create a standard destroy method using the default handler. The handle_attr
* argument must be the attribute specifying the handle to destroy, the
* default handler does not support any other attributes.
*/
#define DECLARE_UVERBS_NAMED_METHOD_DESTROY(_method_id, _handle_attr) \
static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \
_method_id)[] = { _handle_attr }; \
static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \
.id = _method_id, \
.handler = uverbs_destroy_def_handler, \
.num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \
.attrs = &UVERBS_METHOD_ATTRS(_method_id), \
}
#define DECLARE_UVERBS_NAMED_OBJECT(_object_id, _type_attrs, ...) \
static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \
_object_id)[] = { __VA_ARGS__ }; \
static const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \
.id = _object_id, \
.type_attrs = &_type_attrs, \
.num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \
.methods = &UVERBS_OBJECT_METHODS(_object_id) \
}
/*
* Declare global methods. These still have a unique object_id because we
* identify all uapi methods with a (object,method) tuple. However, they have
* no type pointer.
*/
#define DECLARE_UVERBS_GLOBAL_METHODS(_object_id, ...) \
static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \
_object_id)[] = { __VA_ARGS__ }; \
static const struct uverbs_object_def UVERBS_OBJECT(_object_id) = { \
.id = _object_id, \
.num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \
.methods = &UVERBS_OBJECT_METHODS(_object_id) \
}
/* Used by drivers to declare a complete parsing tree for new methods
*/
#define ADD_UVERBS_METHODS(_name, _object_id, ...) \
static const struct uverbs_method_def *const UVERBS_OBJECT_METHODS( \
_object_id)[] = { __VA_ARGS__ }; \
static const struct uverbs_object_def _name = { \
.id = _object_id, \
.num_methods = ARRAY_SIZE(UVERBS_OBJECT_METHODS(_object_id)), \
.methods = &UVERBS_OBJECT_METHODS(_object_id) \
};
/* Used by drivers to declare a complete parsing tree for a single method that
* differs only in having additional driver specific attributes.
*/
#define ADD_UVERBS_ATTRIBUTES_SIMPLE(_name, _object_id, _method_id, ...) \
static const struct uverbs_attr_def *const UVERBS_METHOD_ATTRS( \
_method_id)[] = { __VA_ARGS__ }; \
static const struct uverbs_method_def UVERBS_METHOD(_method_id) = { \
.id = _method_id, \
.num_attrs = ARRAY_SIZE(UVERBS_METHOD_ATTRS(_method_id)), \
.attrs = &UVERBS_METHOD_ATTRS(_method_id), \
}; \
ADD_UVERBS_METHODS(_name, _object_id, &UVERBS_METHOD(_method_id))
#endif

193
sys/ofed/include/rdma/uverbs_std_types.h Normal file
View File

@ -0,0 +1,193 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _UVERBS_STD_TYPES__
#define _UVERBS_STD_TYPES__
#include <rdma/uverbs_types.h>
#include <rdma/uverbs_ioctl.h>
#include <rdma/ib_user_ioctl_verbs.h>
/* Returns _id, or causes a compile error if _id is not a u32.
*
* The uobj APIs should only be used with the write based uAPI to access
* object IDs. The write API must use a u32 for the object handle, which is
* checked by this macro.
*/
#define _uobj_check_id(_id) ({ CTASSERT(sizeof(_id) == sizeof(u32)); (_id); })
#define uobj_get_type(_attrs, _object) \
uapi_get_object((_attrs)->ufile->device->uapi, _object)
#define uobj_get_read(_type, _id, _attrs) \
rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
_uobj_check_id(_id), UVERBS_LOOKUP_READ, \
_attrs)
#define ufd_get_read(_type, _fdnum, _attrs) ({ \
CTASSERT(sizeof(_fdnum) == sizeof(s32)); \
rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
(_fdnum), \
UVERBS_LOOKUP_READ, _attrs); \
})
static inline void *_uobj_get_obj_read(struct ib_uobject *uobj)
{
if (IS_ERR(uobj))
return NULL;
return uobj->object;
}
#define uobj_get_obj_read(_object, _type, _id, _attrs) \
((struct ib_##_object *)_uobj_get_obj_read( \
uobj_get_read(_type, _id, _attrs)))
#define uobj_get_write(_type, _id, _attrs) \
rdma_lookup_get_uobject(uobj_get_type(_attrs, _type), (_attrs)->ufile, \
_uobj_check_id(_id), UVERBS_LOOKUP_WRITE, \
_attrs)
int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
struct uverbs_attr_bundle *attrs);
#define uobj_perform_destroy(_type, _id, _attrs) \
__uobj_perform_destroy(uobj_get_type(_attrs, _type), \
_uobj_check_id(_id), _attrs)
struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
u32 id, struct uverbs_attr_bundle *attrs);
#define uobj_get_destroy(_type, _id, _attrs) \
__uobj_get_destroy(uobj_get_type(_attrs, _type), _uobj_check_id(_id), \
_attrs)
static inline void uobj_put_destroy(struct ib_uobject *uobj)
{
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
}
static inline void uobj_put_read(struct ib_uobject *uobj)
{
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ);
}
#define uobj_put_obj_read(_obj) \
uobj_put_read((_obj)->uobject)
static inline void uobj_put_write(struct ib_uobject *uobj)
{
rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
}
static inline void uobj_alloc_abort(struct ib_uobject *uobj,
struct uverbs_attr_bundle *attrs)
{
rdma_alloc_abort_uobject(uobj, attrs);
}
static inline struct ib_uobject *
__uobj_alloc(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs, struct ib_device **ib_dev)
{
struct ib_uobject *uobj = rdma_alloc_begin_uobject(obj, attrs);
if (!IS_ERR(uobj))
*ib_dev = attrs->context->device;
return uobj;
}
#define uobj_alloc(_type, _attrs, _ib_dev) \
__uobj_alloc(uobj_get_type(_attrs, _type), _attrs, _ib_dev)
static inline void uverbs_flow_action_fill_action(struct ib_flow_action *action,
struct ib_uobject *uobj,
struct ib_device *ib_dev,
enum ib_flow_action_type type)
{
atomic_set(&action->usecnt, 0);
action->device = ib_dev;
action->type = type;
action->uobject = uobj;
uobj->object = action;
}
struct ib_uflow_resources {
size_t max;
size_t num;
size_t collection_num;
size_t counters_num;
struct ib_counters **counters;
struct ib_flow_action **collection;
};
struct ib_uflow_object {
struct ib_uobject uobject;
struct ib_uflow_resources *resources;
};
struct ib_uflow_resources *flow_resources_alloc(size_t num_specs);
void flow_resources_add(struct ib_uflow_resources *uflow_res,
enum ib_flow_spec_type type,
void *ibobj);
void ib_uverbs_flow_resources_free(struct ib_uflow_resources *uflow_res);
static inline void ib_set_flow(struct ib_uobject *uobj, struct ib_flow *ibflow,
struct ib_qp *qp, struct ib_device *device,
struct ib_uflow_resources *uflow_res)
{
struct ib_uflow_object *uflow;
uobj->object = ibflow;
ibflow->uobject = uobj;
if (qp) {
atomic_inc(&qp->usecnt);
ibflow->qp = qp;
}
ibflow->device = device;
uflow = container_of(uobj, typeof(*uflow), uobject);
uflow->resources = uflow_res;
}
struct uverbs_api_object {
const struct uverbs_obj_type *type_attrs;
const struct uverbs_obj_type_class *type_class;
u8 disabled:1;
u32 id;
};
static inline u32 uobj_get_object_id(struct ib_uobject *uobj)
{
return uobj->uapi_object->id;
}
#endif

210
sys/ofed/include/rdma/uverbs_types.h Normal file
View File

@ -0,0 +1,210 @@
/*
* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef _UVERBS_TYPES_
#define _UVERBS_TYPES_
#include <linux/kernel.h>
#include <linux/fs.h>
#include <rdma/ib_verbs.h>
struct uverbs_obj_type;
struct uverbs_api_object;
struct uverbs_attr_bundle;
struct ib_uverbs_file;
enum rdma_remove_reason;
enum rdma_lookup_mode {
UVERBS_LOOKUP_READ,
UVERBS_LOOKUP_WRITE,
/*
* Destroy is like LOOKUP_WRITE, except that the uobject is not
* locked. uobj_destroy is used to convert a LOOKUP_DESTROY lock into
* a LOOKUP_WRITE lock.
*/
UVERBS_LOOKUP_DESTROY,
};
/*
* The following sequences are valid:
* Success flow:
* alloc_begin
* alloc_commit
* [..]
* Access flow:
* lookup_get(exclusive=false) & uverbs_try_lock_object
* lookup_put(exclusive=false) via rdma_lookup_put_uobject
* Destruction flow:
* lookup_get(exclusive=true) & uverbs_try_lock_object
* remove_commit
* remove_handle (optional)
* lookup_put(exclusive=true) via rdma_lookup_put_uobject
*
* Allocate Error flow #1
* alloc_begin
* alloc_abort
* Allocate Error flow #2
* alloc_begin
* remove_commit
* alloc_abort
* Allocate Error flow #3
* alloc_begin
* alloc_commit (fails)
* remove_commit
* alloc_abort
*
* In all cases the caller must hold the ufile kref until alloc_commit or
* alloc_abort returns.
*/
struct uverbs_obj_type_class {
struct ib_uobject *(*alloc_begin)(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs);
/* This consumes the kref on uobj */
void (*alloc_commit)(struct ib_uobject *uobj);
/* This does not consume the kref on uobj */
void (*alloc_abort)(struct ib_uobject *uobj);
struct ib_uobject *(*lookup_get)(const struct uverbs_api_object *obj,
struct ib_uverbs_file *ufile, s64 id,
enum rdma_lookup_mode mode);
void (*lookup_put)(struct ib_uobject *uobj, enum rdma_lookup_mode mode);
/* This does not consume the kref on uobj */
int __must_check (*destroy_hw)(struct ib_uobject *uobj,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs);
void (*remove_handle)(struct ib_uobject *uobj);
};
struct uverbs_obj_type {
const struct uverbs_obj_type_class * const type_class;
size_t obj_size;
};
/*
* Objects type classes which support a detach state (object is still alive but
* it's not attached to any context need to make sure:
* (a) no call through to a driver after a detach is called
* (b) detach isn't called concurrently with context_cleanup
*/
struct uverbs_obj_idr_type {
/*
* In idr based objects, uverbs_obj_type_class points to a generic
* idr operations. In order to specialize the underlying types (e.g. CQ,
* QPs, etc.), we add destroy_object specific callbacks.
*/
struct uverbs_obj_type type;
/* Free driver resources from the uobject, make the driver uncallable,
* and move the uobject to the detached state. If the object was
* destroyed by the user's request, a failure should leave the uobject
* completely unchanged.
*/
int __must_check (*destroy_object)(struct ib_uobject *uobj,
enum rdma_remove_reason why,
struct uverbs_attr_bundle *attrs);
};
struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
struct ib_uverbs_file *ufile, s64 id,
enum rdma_lookup_mode mode,
struct uverbs_attr_bundle *attrs);
void rdma_lookup_put_uobject(struct ib_uobject *uobj,
enum rdma_lookup_mode mode);
struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
struct uverbs_attr_bundle *attrs);
void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
struct uverbs_attr_bundle *attrs);
void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
struct uverbs_attr_bundle *attrs);
/*
* uverbs_uobject_get is called in order to increase the reference count on
* an uobject. This is useful when a handler wants to keep the uobject's memory
* alive, regardless if this uobject is still alive in the context's objects
* repository. Objects are put via uverbs_uobject_put.
*/
static inline void uverbs_uobject_get(struct ib_uobject *uobject)
{
kref_get(&uobject->ref);
}
void uverbs_uobject_put(struct ib_uobject *uobject);
struct uverbs_obj_fd_type {
/*
* In fd based objects, uverbs_obj_type_ops points to generic
* fd operations. In order to specialize the underlying types (e.g.
* completion_channel), we use fops, name and flags for fd creation.
* destroy_object is called when the uobject is to be destroyed,
* because the driver is removed or the FD is closed.
*/
struct uverbs_obj_type type;
int (*destroy_object)(struct ib_uobject *uobj,
enum rdma_remove_reason why);
const struct file_operations *fops;
const char *name;
int flags;
};
extern const struct uverbs_obj_type_class uverbs_idr_class;
extern const struct uverbs_obj_type_class uverbs_fd_class;
int uverbs_uobject_fd_release(struct inode *inode, struct file *filp);
#define UVERBS_BUILD_BUG_ON(cond) (sizeof(char[1 - 2 * !!(cond)]) - \
sizeof(char))
#define UVERBS_TYPE_ALLOC_FD(_obj_size, _destroy_object, _fops, _name, _flags) \
((&((const struct uverbs_obj_fd_type) \
{.type = { \
.type_class = &uverbs_fd_class, \
.obj_size = (_obj_size) + \
UVERBS_BUILD_BUG_ON((_obj_size) < \
sizeof(struct ib_uobject)), \
}, \
.destroy_object = _destroy_object, \
.fops = _fops, \
.name = _name, \
.flags = _flags}))->type)
#define UVERBS_TYPE_ALLOC_IDR_SZ(_size, _destroy_object) \
((&((const struct uverbs_obj_idr_type) \
{.type = { \
.type_class = &uverbs_idr_class, \
.obj_size = (_size) + \
UVERBS_BUILD_BUG_ON((_size) < \
sizeof(struct ib_uobject)) \
}, \
.destroy_object = _destroy_object,}))->type)
#define UVERBS_TYPE_ALLOC_IDR(_destroy_object) \
UVERBS_TYPE_ALLOC_IDR_SZ(sizeof(struct ib_uobject), \
_destroy_object)
#endif

259
sys/ofed/include/uapi/rdma/ib_user_ioctl_cmds.h Normal file
View File

@ -0,0 +1,259 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef IB_USER_IOCTL_CMDS_H
#define IB_USER_IOCTL_CMDS_H
#define UVERBS_ID_NS_MASK 0xF000
#define UVERBS_ID_NS_SHIFT 12
#define UVERBS_UDATA_DRIVER_DATA_NS 1
#define UVERBS_UDATA_DRIVER_DATA_FLAG (1UL << UVERBS_ID_NS_SHIFT)
enum uverbs_default_objects {
UVERBS_OBJECT_DEVICE, /* No instances of DEVICE are allowed */
UVERBS_OBJECT_PD,
UVERBS_OBJECT_COMP_CHANNEL,
UVERBS_OBJECT_CQ,
UVERBS_OBJECT_QP,
UVERBS_OBJECT_SRQ,
UVERBS_OBJECT_AH,
UVERBS_OBJECT_MR,
UVERBS_OBJECT_MW,
UVERBS_OBJECT_FLOW,
UVERBS_OBJECT_XRCD,
UVERBS_OBJECT_RWQ_IND_TBL,
UVERBS_OBJECT_WQ,
UVERBS_OBJECT_FLOW_ACTION,
UVERBS_OBJECT_DM,
UVERBS_OBJECT_COUNTERS,
UVERBS_OBJECT_ASYNC_EVENT,
};
enum {
UVERBS_ATTR_UHW_IN = UVERBS_UDATA_DRIVER_DATA_FLAG,
UVERBS_ATTR_UHW_OUT,
};
enum uverbs_methods_device {
UVERBS_METHOD_INVOKE_WRITE,
UVERBS_METHOD_INFO_HANDLES,
UVERBS_METHOD_QUERY_PORT,
UVERBS_METHOD_GET_CONTEXT,
};
enum uverbs_attrs_invoke_write_cmd_attr_ids {
UVERBS_ATTR_CORE_IN,
UVERBS_ATTR_CORE_OUT,
UVERBS_ATTR_WRITE_CMD,
};
enum uverbs_attrs_query_port_cmd_attr_ids {
UVERBS_ATTR_QUERY_PORT_PORT_NUM,
UVERBS_ATTR_QUERY_PORT_RESP,
};
enum uverbs_attrs_get_context_attr_ids {
UVERBS_ATTR_GET_CONTEXT_NUM_COMP_VECTORS,
UVERBS_ATTR_GET_CONTEXT_CORE_SUPPORT,
};
enum uverbs_attrs_create_cq_cmd_attr_ids {
UVERBS_ATTR_CREATE_CQ_HANDLE,
UVERBS_ATTR_CREATE_CQ_CQE,
UVERBS_ATTR_CREATE_CQ_USER_HANDLE,
UVERBS_ATTR_CREATE_CQ_COMP_CHANNEL,
UVERBS_ATTR_CREATE_CQ_COMP_VECTOR,
UVERBS_ATTR_CREATE_CQ_FLAGS,
UVERBS_ATTR_CREATE_CQ_RESP_CQE,
};
enum uverbs_attrs_destroy_cq_cmd_attr_ids {
UVERBS_ATTR_DESTROY_CQ_HANDLE,
UVERBS_ATTR_DESTROY_CQ_RESP,
};
enum uverbs_attrs_create_flow_action_esp {
UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE,
UVERBS_ATTR_FLOW_ACTION_ESP_ATTRS,
UVERBS_ATTR_FLOW_ACTION_ESP_ESN,
UVERBS_ATTR_FLOW_ACTION_ESP_KEYMAT,
UVERBS_ATTR_FLOW_ACTION_ESP_REPLAY,
UVERBS_ATTR_FLOW_ACTION_ESP_ENCAP,
};
enum uverbs_attrs_modify_flow_action_esp {
UVERBS_ATTR_MODIFY_FLOW_ACTION_ESP_HANDLE =
UVERBS_ATTR_CREATE_FLOW_ACTION_ESP_HANDLE,
};
enum uverbs_attrs_destroy_flow_action_esp {
UVERBS_ATTR_DESTROY_FLOW_ACTION_HANDLE,
};
enum uverbs_methods_cq {
UVERBS_METHOD_CQ_CREATE,
UVERBS_METHOD_CQ_DESTROY,
};
enum uverbs_methods_actions_flow_action_ops {
UVERBS_METHOD_FLOW_ACTION_ESP_CREATE,
UVERBS_METHOD_FLOW_ACTION_DESTROY,
UVERBS_METHOD_FLOW_ACTION_ESP_MODIFY,
};
enum uverbs_attrs_alloc_dm_cmd_attr_ids {
UVERBS_ATTR_ALLOC_DM_HANDLE,
UVERBS_ATTR_ALLOC_DM_LENGTH,
UVERBS_ATTR_ALLOC_DM_ALIGNMENT,
};
enum uverbs_attrs_free_dm_cmd_attr_ids {
UVERBS_ATTR_FREE_DM_HANDLE,
};
enum uverbs_methods_dm {
UVERBS_METHOD_DM_ALLOC,
UVERBS_METHOD_DM_FREE,
};
enum uverbs_attrs_reg_dm_mr_cmd_attr_ids {
UVERBS_ATTR_REG_DM_MR_HANDLE,
UVERBS_ATTR_REG_DM_MR_OFFSET,
UVERBS_ATTR_REG_DM_MR_LENGTH,
UVERBS_ATTR_REG_DM_MR_PD_HANDLE,
UVERBS_ATTR_REG_DM_MR_ACCESS_FLAGS,
UVERBS_ATTR_REG_DM_MR_DM_HANDLE,
UVERBS_ATTR_REG_DM_MR_RESP_LKEY,
UVERBS_ATTR_REG_DM_MR_RESP_RKEY,
};
enum uverbs_methods_mr {
UVERBS_METHOD_DM_MR_REG,
UVERBS_METHOD_MR_DESTROY,
UVERBS_METHOD_ADVISE_MR,
};
enum uverbs_attrs_mr_destroy_ids {
UVERBS_ATTR_DESTROY_MR_HANDLE,
};
enum uverbs_attrs_advise_mr_cmd_attr_ids {
UVERBS_ATTR_ADVISE_MR_PD_HANDLE,
UVERBS_ATTR_ADVISE_MR_ADVICE,
UVERBS_ATTR_ADVISE_MR_FLAGS,
UVERBS_ATTR_ADVISE_MR_SGE_LIST,
};
enum uverbs_attrs_create_counters_cmd_attr_ids {
UVERBS_ATTR_CREATE_COUNTERS_HANDLE,
};
enum uverbs_attrs_destroy_counters_cmd_attr_ids {
UVERBS_ATTR_DESTROY_COUNTERS_HANDLE,
};
enum uverbs_attrs_read_counters_cmd_attr_ids {
UVERBS_ATTR_READ_COUNTERS_HANDLE,
UVERBS_ATTR_READ_COUNTERS_BUFF,
UVERBS_ATTR_READ_COUNTERS_FLAGS,
};
enum uverbs_methods_actions_counters_ops {
UVERBS_METHOD_COUNTERS_CREATE,
UVERBS_METHOD_COUNTERS_DESTROY,
UVERBS_METHOD_COUNTERS_READ,
};
enum uverbs_attrs_info_handles_id {
UVERBS_ATTR_INFO_OBJECT_ID,
UVERBS_ATTR_INFO_TOTAL_HANDLES,
UVERBS_ATTR_INFO_HANDLES_LIST,
};
enum uverbs_methods_pd {
UVERBS_METHOD_PD_DESTROY,
};
enum uverbs_attrs_pd_destroy_ids {
UVERBS_ATTR_DESTROY_PD_HANDLE,
};
enum uverbs_methods_mw {
UVERBS_METHOD_MW_DESTROY,
};
enum uverbs_attrs_mw_destroy_ids {
UVERBS_ATTR_DESTROY_MW_HANDLE,
};
enum uverbs_methods_xrcd {
UVERBS_METHOD_XRCD_DESTROY,
};
enum uverbs_attrs_xrcd_destroy_ids {
UVERBS_ATTR_DESTROY_XRCD_HANDLE,
};
enum uverbs_methods_ah {
UVERBS_METHOD_AH_DESTROY,
};
enum uverbs_attrs_ah_destroy_ids {
UVERBS_ATTR_DESTROY_AH_HANDLE,
};
enum uverbs_methods_rwq_ind_tbl {
UVERBS_METHOD_RWQ_IND_TBL_DESTROY,
};
enum uverbs_attrs_rwq_ind_tbl_destroy_ids {
UVERBS_ATTR_DESTROY_RWQ_IND_TBL_HANDLE,
};
enum uverbs_methods_flow {
UVERBS_METHOD_FLOW_DESTROY,
};
enum uverbs_attrs_flow_destroy_ids {
UVERBS_ATTR_DESTROY_FLOW_HANDLE,
};
enum uverbs_method_async_event {
UVERBS_METHOD_ASYNC_EVENT_ALLOC,
};
enum uverbs_attrs_async_event_create {
UVERBS_ATTR_ASYNC_EVENT_ALLOC_FD_HANDLE,
};
#endif

211
sys/ofed/include/uapi/rdma/ib_user_ioctl_verbs.h Normal file
View File

@ -0,0 +1,211 @@
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
/*
* Copyright (c) 2017-2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef IB_USER_IOCTL_VERBS_H
#define IB_USER_IOCTL_VERBS_H
#include <linux/types.h>
#include <rdma/ib_user_verbs.h>
#ifndef RDMA_UAPI_PTR
#define RDMA_UAPI_PTR(_type, _name) __aligned_u64 _name
#endif
#define IB_UVERBS_ACCESS_OPTIONAL_FIRST (1 << 20)
#define IB_UVERBS_ACCESS_OPTIONAL_LAST (1 << 29)
enum ib_uverbs_core_support {
IB_UVERBS_CORE_SUPPORT_OPTIONAL_MR_ACCESS = 1 << 0,
};
enum ib_uverbs_access_flags {
IB_UVERBS_ACCESS_LOCAL_WRITE = 1 << 0,
IB_UVERBS_ACCESS_REMOTE_WRITE = 1 << 1,
IB_UVERBS_ACCESS_REMOTE_READ = 1 << 2,
IB_UVERBS_ACCESS_REMOTE_ATOMIC = 1 << 3,
IB_UVERBS_ACCESS_MW_BIND = 1 << 4,
IB_UVERBS_ACCESS_ZERO_BASED = 1 << 5,
IB_UVERBS_ACCESS_ON_DEMAND = 1 << 6,
IB_UVERBS_ACCESS_HUGETLB = 1 << 7,
IB_UVERBS_ACCESS_RELAXED_ORDERING = IB_UVERBS_ACCESS_OPTIONAL_FIRST,
IB_UVERBS_ACCESS_OPTIONAL_RANGE =
((IB_UVERBS_ACCESS_OPTIONAL_LAST << 1) - 1) &
~(IB_UVERBS_ACCESS_OPTIONAL_FIRST - 1)
};
enum ib_uverbs_query_port_cap_flags {
IB_UVERBS_PCF_SM = 1 << 1,
IB_UVERBS_PCF_NOTICE_SUP = 1 << 2,
IB_UVERBS_PCF_TRAP_SUP = 1 << 3,
IB_UVERBS_PCF_OPT_IPD_SUP = 1 << 4,
IB_UVERBS_PCF_AUTO_MIGR_SUP = 1 << 5,
IB_UVERBS_PCF_SL_MAP_SUP = 1 << 6,
IB_UVERBS_PCF_MKEY_NVRAM = 1 << 7,
IB_UVERBS_PCF_PKEY_NVRAM = 1 << 8,
IB_UVERBS_PCF_LED_INFO_SUP = 1 << 9,
IB_UVERBS_PCF_SM_DISABLED = 1 << 10,
IB_UVERBS_PCF_SYS_IMAGE_GUID_SUP = 1 << 11,
IB_UVERBS_PCF_PKEY_SW_EXT_PORT_TRAP_SUP = 1 << 12,
IB_UVERBS_PCF_EXTENDED_SPEEDS_SUP = 1 << 14,
IB_UVERBS_PCF_CM_SUP = 1 << 16,
IB_UVERBS_PCF_SNMP_TUNNEL_SUP = 1 << 17,
IB_UVERBS_PCF_REINIT_SUP = 1 << 18,
IB_UVERBS_PCF_DEVICE_MGMT_SUP = 1 << 19,
IB_UVERBS_PCF_VENDOR_CLASS_SUP = 1 << 20,
IB_UVERBS_PCF_DR_NOTICE_SUP = 1 << 21,
IB_UVERBS_PCF_CAP_MASK_NOTICE_SUP = 1 << 22,
IB_UVERBS_PCF_BOOT_MGMT_SUP = 1 << 23,
IB_UVERBS_PCF_LINK_LATENCY_SUP = 1 << 24,
IB_UVERBS_PCF_CLIENT_REG_SUP = 1 << 25,
/*
* IsOtherLocalChangesNoticeSupported is aliased by IP_BASED_GIDS and
* is inaccessible
*/
IB_UVERBS_PCF_LINK_SPEED_WIDTH_TABLE_SUP = 1 << 27,
IB_UVERBS_PCF_VENDOR_SPECIFIC_MADS_TABLE_SUP = 1 << 28,
IB_UVERBS_PCF_MCAST_PKEY_TRAP_SUPPRESSION_SUP = 1 << 29,
IB_UVERBS_PCF_MCAST_FDB_TOP_SUP = 1 << 30,
IB_UVERBS_PCF_HIERARCHY_INFO_SUP = 1ULL << 31,
/* NOTE this is an internal flag, not an IBA flag */
IB_UVERBS_PCF_IP_BASED_GIDS = 1 << 26,
};
enum ib_uverbs_query_port_flags {
IB_UVERBS_QPF_GRH_REQUIRED = 1 << 0,
};
enum ib_uverbs_flow_action_esp_keymat {
IB_UVERBS_FLOW_ACTION_ESP_KEYMAT_AES_GCM,
};
enum ib_uverbs_flow_action_esp_keymat_aes_gcm_iv_algo {
IB_UVERBS_FLOW_ACTION_IV_ALGO_SEQ,
};
struct ib_uverbs_flow_action_esp_keymat_aes_gcm {
__aligned_u64 iv;
__u32 iv_algo; /* Use enum ib_uverbs_flow_action_esp_keymat_aes_gcm_iv_algo */
__u32 salt;
__u32 icv_len;
__u32 key_len;
__u32 aes_key[256 / 32];
};
enum ib_uverbs_flow_action_esp_replay {
IB_UVERBS_FLOW_ACTION_ESP_REPLAY_NONE,
IB_UVERBS_FLOW_ACTION_ESP_REPLAY_BMP,
};
struct ib_uverbs_flow_action_esp_replay_bmp {
__u32 size;
};
enum ib_uverbs_flow_action_esp_flags {
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_INLINE_CRYPTO = 0UL << 0, /* Default */
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_FULL_OFFLOAD = 1UL << 0,
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_TUNNEL = 0UL << 1, /* Default */
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_TRANSPORT = 1UL << 1,
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_DECRYPT = 0UL << 2, /* Default */
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ENCRYPT = 1UL << 2,
IB_UVERBS_FLOW_ACTION_ESP_FLAGS_ESN_NEW_WINDOW = 1UL << 3,
};
struct ib_uverbs_flow_action_esp_encap {
/* This struct represents a list of pointers to flow_xxxx_filter that
* encapsulates the payload in ESP tunnel mode.
*/
RDMA_UAPI_PTR(void *, val_ptr); /* pointer to a flow_xxxx_filter */
RDMA_UAPI_PTR(struct ib_uverbs_flow_action_esp_encap *, next_ptr);
__u16 len; /* Len of the filter struct val_ptr points to */
__u16 type; /* Use flow_spec_type enum */
};
struct ib_uverbs_flow_action_esp {
__u32 spi;
__u32 seq;
__u32 tfc_pad;
__u32 flags;
__aligned_u64 hard_limit_pkts;
};
enum ib_uverbs_read_counters_flags {
/* prefer read values from driver cache */
IB_UVERBS_READ_COUNTERS_PREFER_CACHED = 1 << 0,
};
enum ib_uverbs_advise_mr_advice {
IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH,
IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE,
};
enum ib_uverbs_advise_mr_flag {
IB_UVERBS_ADVISE_MR_FLAG_FLUSH = 1 << 0,
};
struct ib_uverbs_query_port_resp_ex {
struct ib_uverbs_query_port_resp legacy_resp;
__u16 port_cap_flags2;
__u8 reserved[6];
};
enum rdma_driver_id {
RDMA_DRIVER_UNKNOWN,
RDMA_DRIVER_MLX5,
RDMA_DRIVER_MLX4,
RDMA_DRIVER_CXGB3,
RDMA_DRIVER_CXGB4,
RDMA_DRIVER_MTHCA,
RDMA_DRIVER_BNXT_RE,
RDMA_DRIVER_OCRDMA,
RDMA_DRIVER_NES,
RDMA_DRIVER_I40IW,
RDMA_DRIVER_VMW_PVRDMA,
RDMA_DRIVER_QEDR,
RDMA_DRIVER_HNS,
RDMA_DRIVER_USNIC,
RDMA_DRIVER_RXE,
RDMA_DRIVER_HFI1,
RDMA_DRIVER_QIB,
RDMA_DRIVER_EFA,
RDMA_DRIVER_SIW,
RDMA_DRIVER_QLNXR,
};
#endif

20
sys/ofed/include/uapi/rdma/ib_user_mad.h
View File

@ -38,13 +38,7 @@
#ifndef IB_USER_MAD_H
#define IB_USER_MAD_H
#ifdef _KERNEL
#include <linux/types.h>
#include <linux/ioctl.h>
#else
#include <infiniband/types.h>
#include <sys/ioccom.h>
#endif
#include <rdma/rdma_user_ioctl.h>
/*
* Increment this value if any changes that break userspace ABI
@ -239,16 +233,4 @@ struct ib_user_mad_reg_req2 {
__u8 reserved[3];
};
#define IB_IOCTL_MAGIC 0x1b
#define IB_USER_MAD_REGISTER_AGENT _IOWR(IB_IOCTL_MAGIC, 1, \
struct ib_user_mad_reg_req)
#define IB_USER_MAD_UNREGISTER_AGENT _IOW(IB_IOCTL_MAGIC, 2, __u32)
#define IB_USER_MAD_ENABLE_PKEY _IO(IB_IOCTL_MAGIC, 3)
#define IB_USER_MAD_REGISTER_AGENT2 _IOWR(IB_IOCTL_MAGIC, 4, \
struct ib_user_mad_reg_req2)
#endif /* IB_USER_MAD_H */

424
sys/ofed/include/uapi/rdma/ib_user_verbs.h
View File

@ -53,7 +53,7 @@
#define IB_USER_VERBS_ABI_VERSION 6
#define IB_USER_VERBS_CMD_THRESHOLD 50
enum {
enum ib_uverbs_write_cmds {
IB_USER_VERBS_CMD_GET_CONTEXT,
IB_USER_VERBS_CMD_QUERY_DEVICE,
IB_USER_VERBS_CMD_QUERY_PORT,
@ -101,13 +101,15 @@ enum {
IB_USER_VERBS_EX_CMD_QUERY_DEVICE = IB_USER_VERBS_CMD_QUERY_DEVICE,
IB_USER_VERBS_EX_CMD_CREATE_CQ = IB_USER_VERBS_CMD_CREATE_CQ,
IB_USER_VERBS_EX_CMD_CREATE_QP = IB_USER_VERBS_CMD_CREATE_QP,
IB_USER_VERBS_EX_CMD_MODIFY_QP = IB_USER_VERBS_CMD_MODIFY_QP,
IB_USER_VERBS_EX_CMD_CREATE_FLOW = IB_USER_VERBS_CMD_THRESHOLD,
IB_USER_VERBS_EX_CMD_DESTROY_FLOW,
IB_USER_VERBS_EX_CMD_CREATE_WQ,
IB_USER_VERBS_EX_CMD_MODIFY_WQ,
IB_USER_VERBS_EX_CMD_DESTROY_WQ,
IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL,
IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL
IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL,
IB_USER_VERBS_EX_CMD_MODIFY_CQ
};
/*
@ -122,13 +124,19 @@ enum {
*/
struct ib_uverbs_async_event_desc {
__u64 element;
__aligned_u64 element;
__u32 event_type; /* enum ib_event_type */
__u32 reserved;
};
struct ib_uverbs_comp_event_desc {
__u64 cq_handle;
__aligned_u64 cq_handle;
};
struct ib_uverbs_cq_moderation_caps {
__u16 max_cq_moderation_count;
__u16 max_cq_moderation_period;
__u32 reserved;
};
/*
@ -140,10 +148,7 @@ struct ib_uverbs_comp_event_desc {
*/
#define IB_USER_VERBS_CMD_COMMAND_MASK 0xff
#define IB_USER_VERBS_CMD_FLAGS_MASK 0xff000000u
#define IB_USER_VERBS_CMD_FLAGS_SHIFT 24
#define IB_USER_VERBS_CMD_FLAG_EXTENDED 0x80
#define IB_USER_VERBS_CMD_FLAG_EXTENDED 0x80000000u
struct ib_uverbs_cmd_hdr {
__u32 command;
@ -152,33 +157,34 @@ struct ib_uverbs_cmd_hdr {
};
struct ib_uverbs_ex_cmd_hdr {
__u64 response;
__aligned_u64 response;
__u16 provider_in_words;
__u16 provider_out_words;
__u32 cmd_hdr_reserved;
};
struct ib_uverbs_get_context {
__u64 response;
__u64 driver_data[0];
__aligned_u64 response;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_get_context_resp {
__u32 async_fd;
__u32 num_comp_vectors;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_device {
__u64 response;
__u64 driver_data[0];
__aligned_u64 response;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_device_resp {
__u64 fw_ver;
__aligned_u64 fw_ver;
__be64 node_guid;
__be64 sys_image_guid;
__u64 max_mr_size;
__u64 page_size_cap;
__aligned_u64 max_mr_size;
__aligned_u64 page_size_cap;
__u32 vendor_id;
__u32 vendor_part_id;
__u32 hw_ver;
@ -223,7 +229,7 @@ struct ib_uverbs_ex_query_device {
};
struct ib_uverbs_odp_caps {
__u64 general_caps;
__aligned_u64 general_caps;
struct {
__u32 rc_odp_caps;
__u32 uc_odp_caps;
@ -243,28 +249,47 @@ struct ib_uverbs_rss_caps {
__u32 reserved;
};
struct ib_uverbs_tm_caps {
/* Max size of rendezvous request message */
__u32 max_rndv_hdr_size;
/* Max number of entries in tag matching list */
__u32 max_num_tags;
/* TM flags */
__u32 flags;
/* Max number of outstanding list operations */
__u32 max_ops;
/* Max number of SGE in tag matching entry */
__u32 max_sge;
__u32 reserved;
};
struct ib_uverbs_ex_query_device_resp {
struct ib_uverbs_query_device_resp base;
__u32 comp_mask;
__u32 response_length;
struct ib_uverbs_odp_caps odp_caps;
__u64 timestamp_mask;
__u64 hca_core_clock; /* in KHZ */
__u64 device_cap_flags_ex;
__aligned_u64 timestamp_mask;
__aligned_u64 hca_core_clock; /* in KHZ */
__aligned_u64 device_cap_flags_ex;
struct ib_uverbs_rss_caps rss_caps;
__u32 max_wq_type_rq;
__u32 raw_packet_caps;
struct ib_uverbs_tm_caps tm_caps;
struct ib_uverbs_cq_moderation_caps cq_moderation_caps;
__aligned_u64 max_dm_size;
__u32 xrc_odp_caps;
__u32 reserved;
};
struct ib_uverbs_query_port {
__u64 response;
__aligned_u64 response;
__u8 port_num;
__u8 reserved[7];
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_port_resp {
__u32 port_cap_flags;
__u32 port_cap_flags; /* see ib_uverbs_query_port_cap_flags */
__u32 max_msg_sz;
__u32 bad_pkey_cntr;
__u32 qkey_viol_cntr;
@ -284,16 +309,18 @@ struct ib_uverbs_query_port_resp {
__u8 active_speed;
__u8 phys_state;
__u8 link_layer;
__u8 reserved[2];
__u8 flags; /* see ib_uverbs_query_port_flags */
__u8 reserved;
};
struct ib_uverbs_alloc_pd {
__u64 response;
__u64 driver_data[0];
__aligned_u64 response;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_alloc_pd_resp {
__u32 pd_handle;
__u32 driver_data[0];
};
struct ib_uverbs_dealloc_pd {
@ -301,14 +328,15 @@ struct ib_uverbs_dealloc_pd {
};
struct ib_uverbs_open_xrcd {
__u64 response;
__aligned_u64 response;
__u32 fd;
__u32 oflags;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_open_xrcd_resp {
__u32 xrcd_handle;
__u32 driver_data[0];
};
struct ib_uverbs_close_xrcd {
@ -316,35 +344,38 @@ struct ib_uverbs_close_xrcd {
};
struct ib_uverbs_reg_mr {
__u64 response;
__u64 start;
__u64 length;
__u64 hca_va;
__aligned_u64 response;
__aligned_u64 start;
__aligned_u64 length;
__aligned_u64 hca_va;
__u32 pd_handle;
__u32 access_flags;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_reg_mr_resp {
__u32 mr_handle;
__u32 lkey;
__u32 rkey;
__u32 driver_data[0];
};
struct ib_uverbs_rereg_mr {
__u64 response;
__aligned_u64 response;
__u32 mr_handle;
__u32 flags;
__u64 start;
__u64 length;
__u64 hca_va;
__aligned_u64 start;
__aligned_u64 length;
__aligned_u64 hca_va;
__u32 pd_handle;
__u32 access_flags;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_rereg_mr_resp {
__u32 lkey;
__u32 rkey;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_dereg_mr {
@ -352,15 +383,17 @@ struct ib_uverbs_dereg_mr {
};
struct ib_uverbs_alloc_mw {
__u64 response;
__aligned_u64 response;
__u32 pd_handle;
__u8 mw_type;
__u8 reserved[3];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_alloc_mw_resp {
__u32 mw_handle;
__u32 rkey;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_dealloc_mw {
@ -368,7 +401,7 @@ struct ib_uverbs_dealloc_mw {
};
struct ib_uverbs_create_comp_channel {
__u64 response;
__aligned_u64 response;
};
struct ib_uverbs_create_comp_channel_resp {
@ -376,28 +409,34 @@ struct ib_uverbs_create_comp_channel_resp {
};
struct ib_uverbs_create_cq {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 cqe;
__u32 comp_vector;
__s32 comp_channel;
__u32 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
enum ib_uverbs_ex_create_cq_flags {
IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION = 1 << 0,
IB_UVERBS_CQ_FLAGS_IGNORE_OVERRUN = 1 << 1,
};
struct ib_uverbs_ex_create_cq {
__u64 user_handle;
__aligned_u64 user_handle;
__u32 cqe;
__u32 comp_vector;
__s32 comp_channel;
__u32 comp_mask;
__u32 flags;
__u32 flags; /* bitmask of ib_uverbs_ex_create_cq_flags */
__u32 reserved;
};
struct ib_uverbs_create_cq_resp {
__u32 cq_handle;
__u32 cqe;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_ex_create_cq_resp {
@ -407,32 +446,32 @@ struct ib_uverbs_ex_create_cq_resp {
};
struct ib_uverbs_resize_cq {
__u64 response;
__aligned_u64 response;
__u32 cq_handle;
__u32 cqe;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_resize_cq_resp {
__u32 cqe;
__u32 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_poll_cq {
__u64 response;
__aligned_u64 response;
__u32 cq_handle;
__u32 ne;
};
struct ib_uverbs_wc {
__u64 wr_id;
__aligned_u64 wr_id;
__u32 status;
__u32 opcode;
__u32 vendor_err;
__u32 byte_len;
union {
__u32 imm_data;
__be32 imm_data;
__u32 invalidate_rkey;
} ex;
__u32 qp_num;
@ -458,7 +497,7 @@ struct ib_uverbs_req_notify_cq {
};
struct ib_uverbs_destroy_cq {
__u64 response;
__aligned_u64 response;
__u32 cq_handle;
__u32 reserved;
};
@ -527,8 +566,8 @@ struct ib_uverbs_qp_attr {
};
struct ib_uverbs_create_qp {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 send_cq_handle;
__u32 recv_cq_handle;
@ -542,7 +581,7 @@ struct ib_uverbs_create_qp {
__u8 qp_type;
__u8 is_srq;
__u8 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
enum ib_uverbs_create_qp_mask {
@ -553,8 +592,22 @@ enum {
IB_UVERBS_CREATE_QP_SUP_COMP_MASK = IB_UVERBS_CREATE_QP_MASK_IND_TABLE,
};
enum {
/*
* This value is equal to IB_QP_DEST_QPN.
*/
IB_USER_LEGACY_LAST_QP_ATTR_MASK = 1ULL << 20,
};
enum {
/*
* This value is equal to IB_QP_RATE_LIMIT.
*/
IB_USER_LAST_QP_ATTR_MASK = 1ULL << 25,
};
struct ib_uverbs_ex_create_qp {
__u64 user_handle;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 send_cq_handle;
__u32 recv_cq_handle;
@ -571,17 +624,17 @@ struct ib_uverbs_ex_create_qp {
__u32 comp_mask;
__u32 create_flags;
__u32 rwq_ind_tbl_handle;
__u32 reserved1;
__u32 source_qpn;
};
struct ib_uverbs_open_qp {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 qpn;
__u8 qp_type;
__u8 reserved[7];
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
/* also used for open response */
@ -594,6 +647,7 @@ struct ib_uverbs_create_qp_resp {
__u32 max_recv_sge;
__u32 max_inline_data;
__u32 reserved;
__u32 driver_data[0];
};
struct ib_uverbs_ex_create_qp_resp {
@ -622,10 +676,10 @@ struct ib_uverbs_qp_dest {
};
struct ib_uverbs_query_qp {
__u64 response;
__aligned_u64 response;
__u32 qp_handle;
__u32 attr_mask;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_qp_resp {
@ -659,7 +713,7 @@ struct ib_uverbs_query_qp_resp {
__u8 alt_timeout;
__u8 sq_sig_all;
__u8 reserved[5];
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_modify_qp {
@ -689,14 +743,22 @@ struct ib_uverbs_modify_qp {
__u8 alt_port_num;
__u8 alt_timeout;
__u8 reserved[2];
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_modify_qp_resp {
struct ib_uverbs_ex_modify_qp {
struct ib_uverbs_modify_qp base;
__u32 rate_limit;
__u32 reserved;
};
struct ib_uverbs_ex_modify_qp_resp {
__u32 comp_mask;
__u32 response_length;
};
struct ib_uverbs_destroy_qp {
__u64 response;
__aligned_u64 response;
__u32 qp_handle;
__u32 reserved;
};
@ -712,30 +774,48 @@ struct ib_uverbs_destroy_qp_resp {
* document the ABI.
*/
struct ib_uverbs_sge {
__u64 addr;
__aligned_u64 addr;
__u32 length;
__u32 lkey;
};
enum ib_uverbs_wr_opcode {
IB_UVERBS_WR_RDMA_WRITE = 0,
IB_UVERBS_WR_RDMA_WRITE_WITH_IMM = 1,
IB_UVERBS_WR_SEND = 2,
IB_UVERBS_WR_SEND_WITH_IMM = 3,
IB_UVERBS_WR_RDMA_READ = 4,
IB_UVERBS_WR_ATOMIC_CMP_AND_SWP = 5,
IB_UVERBS_WR_ATOMIC_FETCH_AND_ADD = 6,
IB_UVERBS_WR_LOCAL_INV = 7,
IB_UVERBS_WR_BIND_MW = 8,
IB_UVERBS_WR_SEND_WITH_INV = 9,
IB_UVERBS_WR_TSO = 10,
IB_UVERBS_WR_RDMA_READ_WITH_INV = 11,
IB_UVERBS_WR_MASKED_ATOMIC_CMP_AND_SWP = 12,
IB_UVERBS_WR_MASKED_ATOMIC_FETCH_AND_ADD = 13,
/* Review enum ib_wr_opcode before modifying this */
};
struct ib_uverbs_send_wr {
__u64 wr_id;
__aligned_u64 wr_id;
__u32 num_sge;
__u32 opcode;
__u32 opcode; /* see enum ib_uverbs_wr_opcode */
__u32 send_flags;
union {
__u32 imm_data;
__be32 imm_data;
__u32 invalidate_rkey;
} ex;
union {
struct {
__u64 remote_addr;
__aligned_u64 remote_addr;
__u32 rkey;
__u32 reserved;
} rdma;
struct {
__u64 remote_addr;
__u64 compare_add;
__u64 swap;
__aligned_u64 remote_addr;
__aligned_u64 compare_add;
__aligned_u64 swap;
__u32 rkey;
__u32 reserved;
} atomic;
@ -749,7 +829,7 @@ struct ib_uverbs_send_wr {
};
struct ib_uverbs_post_send {
__u64 response;
__aligned_u64 response;
__u32 qp_handle;
__u32 wr_count;
__u32 sge_count;
@ -762,13 +842,13 @@ struct ib_uverbs_post_send_resp {
};
struct ib_uverbs_recv_wr {
__u64 wr_id;
__aligned_u64 wr_id;
__u32 num_sge;
__u32 reserved;
};
struct ib_uverbs_post_recv {
__u64 response;
__aligned_u64 response;
__u32 qp_handle;
__u32 wr_count;
__u32 sge_count;
@ -781,7 +861,7 @@ struct ib_uverbs_post_recv_resp {
};
struct ib_uverbs_post_srq_recv {
__u64 response;
__aligned_u64 response;
__u32 srq_handle;
__u32 wr_count;
__u32 sge_count;
@ -794,15 +874,17 @@ struct ib_uverbs_post_srq_recv_resp {
};
struct ib_uverbs_create_ah {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 reserved;
struct ib_uverbs_ah_attr attr;
__aligned_u64 driver_data[0];
};
struct ib_uverbs_create_ah_resp {
__u32 ah_handle;
__u32 driver_data[0];
};
struct ib_uverbs_destroy_ah {
@ -814,7 +896,7 @@ struct ib_uverbs_attach_mcast {
__u32 qp_handle;
__u16 mlid;
__u16 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_detach_mcast {
@ -822,7 +904,7 @@ struct ib_uverbs_detach_mcast {
__u32 qp_handle;
__u16 mlid;
__u16 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_flow_spec_hdr {
@ -830,7 +912,7 @@ struct ib_uverbs_flow_spec_hdr {
__u16 size;
__u16 reserved;
/* followed by flow_spec */
__u64 flow_spec_data[0];
__aligned_u64 flow_spec_data[0];
};
struct ib_uverbs_flow_eth_filter {
@ -916,6 +998,141 @@ struct ib_uverbs_flow_spec_ipv6 {
struct ib_uverbs_flow_ipv6_filter mask;
};
struct ib_uverbs_flow_spec_action_tag {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
__u32 tag_id;
__u32 reserved1;
};
struct ib_uverbs_flow_spec_action_drop {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
};
struct ib_uverbs_flow_spec_action_handle {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
__u32 handle;
__u32 reserved1;
};
struct ib_uverbs_flow_spec_action_count {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
__u32 handle;
__u32 reserved1;
};
struct ib_uverbs_flow_tunnel_filter {
__be32 tunnel_id;
};
struct ib_uverbs_flow_spec_tunnel {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
struct ib_uverbs_flow_tunnel_filter val;
struct ib_uverbs_flow_tunnel_filter mask;
};
struct ib_uverbs_flow_spec_esp_filter {
__u32 spi;
__u32 seq;
};
struct ib_uverbs_flow_spec_esp {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
struct ib_uverbs_flow_spec_esp_filter val;
struct ib_uverbs_flow_spec_esp_filter mask;
};
struct ib_uverbs_flow_gre_filter {
/* c_ks_res0_ver field is bits 0-15 in offset 0 of a standard GRE header:
* bit 0 - C - checksum bit.
* bit 1 - reserved. set to 0.
* bit 2 - key bit.
* bit 3 - sequence number bit.
* bits 4:12 - reserved. set to 0.
* bits 13:15 - GRE version.
*/
__be16 c_ks_res0_ver;
__be16 protocol;
__be32 key;
};
struct ib_uverbs_flow_spec_gre {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
struct ib_uverbs_flow_gre_filter val;
struct ib_uverbs_flow_gre_filter mask;
};
struct ib_uverbs_flow_mpls_filter {
/* The field includes the entire MPLS label:
* bits 0:19 - label field.
* bits 20:22 - traffic class field.
* bits 23 - bottom of stack bit.
* bits 24:31 - ttl field.
*/
__be32 label;
};
struct ib_uverbs_flow_spec_mpls {
union {
struct ib_uverbs_flow_spec_hdr hdr;
struct {
__u32 type;
__u16 size;
__u16 reserved;
};
};
struct ib_uverbs_flow_mpls_filter val;
struct ib_uverbs_flow_mpls_filter mask;
};
struct ib_uverbs_flow_attr {
__u32 type;
__u16 size;
@ -948,27 +1165,27 @@ struct ib_uverbs_destroy_flow {
};
struct ib_uverbs_create_srq {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 max_wr;
__u32 max_sge;
__u32 srq_limit;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_create_xsrq {
__u64 response;
__u64 user_handle;
__aligned_u64 response;
__aligned_u64 user_handle;
__u32 srq_type;
__u32 pd_handle;
__u32 max_wr;
__u32 max_sge;
__u32 srq_limit;
__u32 reserved;
__u32 max_num_tags;
__u32 xrcd_handle;
__u32 cq_handle;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_create_srq_resp {
@ -976,6 +1193,7 @@ struct ib_uverbs_create_srq_resp {
__u32 max_wr;
__u32 max_sge;
__u32 srqn;
__u32 driver_data[0];
};
struct ib_uverbs_modify_srq {
@ -983,14 +1201,14 @@ struct ib_uverbs_modify_srq {
__u32 attr_mask;
__u32 max_wr;
__u32 srq_limit;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_srq {
__u64 response;
__aligned_u64 response;
__u32 srq_handle;
__u32 reserved;
__u64 driver_data[0];
__aligned_u64 driver_data[0];
};
struct ib_uverbs_query_srq_resp {
@ -1001,7 +1219,7 @@ struct ib_uverbs_query_srq_resp {
};
struct ib_uverbs_destroy_srq {
__u64 response;
__aligned_u64 response;
__u32 srq_handle;
__u32 reserved;
};
@ -1013,11 +1231,13 @@ struct ib_uverbs_destroy_srq_resp {
struct ib_uverbs_ex_create_wq {
__u32 comp_mask;
__u32 wq_type;
__u64 user_handle;
__aligned_u64 user_handle;
__u32 pd_handle;
__u32 cq_handle;
__u32 max_wr;
__u32 max_sge;
__u32 create_flags; /* Use enum ib_wq_flags */
__u32 reserved;
};
struct ib_uverbs_ex_create_wq_resp {
@ -1046,6 +1266,8 @@ struct ib_uverbs_ex_modify_wq {
__u32 wq_handle;
__u32 wq_state;
__u32 curr_wq_state;
__u32 flags; /* Use enum ib_wq_flags */
__u32 flags_mask; /* Use enum ib_wq_flags */
};
/* Prevent memory allocation rather than max expected size */
@ -1072,4 +1294,18 @@ struct ib_uverbs_ex_destroy_rwq_ind_table {
__u32 ind_tbl_handle;
};
struct ib_uverbs_cq_moderation {
__u16 cq_count;
__u16 cq_period;
};
struct ib_uverbs_ex_modify_cq {
__u32 cq_handle;
__u32 attr_mask;
struct ib_uverbs_cq_moderation attr;
__u32 reserved;
};
#define IB_DEVICE_NAME_MAX 64
#endif /* IB_USER_VERBS_H */

18
sys/ofed/include/uapi/rdma/mlx5-abi.h
View File

@ -281,4 +281,22 @@ struct mlx5_ib_modify_wq {
__u32 comp_mask;
__u32 reserved;
};
enum mlx5_ib_mmap_cmd {
MLX5_IB_MMAP_REGULAR_PAGE = 0,
MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES = 1,
MLX5_IB_MMAP_WC_PAGE = 2,
MLX5_IB_MMAP_NC_PAGE = 3,
/* 5 is chosen in order to be compatible with old versions of libmlx5 */
MLX5_IB_MMAP_CORE_CLOCK = 5,
MLX5_IB_MMAP_ALLOC_WC = 6,
MLX5_IB_MMAP_CLOCK_INFO = 7,
MLX5_IB_MMAP_DEVICE_MEM = 8,
};
/* Bit indexes for the mlx5_alloc_ucontext_resp.clock_info_versions bitmap */
enum {
MLX5_IB_CLOCK_INFO_V1 = 0,
};
#endif /* MLX5_ABI_USER_H */

283
sys/ofed/include/uapi/rdma/mlx5_user_ioctl_cmds.h Normal file
View File

@ -0,0 +1,283 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef MLX5_USER_IOCTL_CMDS_H
#define MLX5_USER_IOCTL_CMDS_H
#include <linux/types.h>
#include <rdma/ib_user_ioctl_cmds.h>
enum mlx5_ib_create_flow_action_attrs {
/* This attribute belong to the driver namespace */
MLX5_IB_ATTR_CREATE_FLOW_ACTION_FLAGS = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_alloc_dm_attrs {
MLX5_IB_ATTR_ALLOC_DM_RESP_START_OFFSET = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_ALLOC_DM_RESP_PAGE_INDEX,
MLX5_IB_ATTR_ALLOC_DM_REQ_TYPE,
};
enum mlx5_ib_devx_methods {
MLX5_IB_METHOD_DEVX_OTHER = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_DEVX_QUERY_UAR,
MLX5_IB_METHOD_DEVX_QUERY_EQN,
MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT,
};
enum mlx5_ib_devx_other_attrs {
MLX5_IB_ATTR_DEVX_OTHER_CMD_IN = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
};
enum mlx5_ib_devx_obj_create_attrs {
MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
};
enum mlx5_ib_devx_query_uar_attrs {
MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
};
enum mlx5_ib_devx_obj_destroy_attrs {
MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_devx_obj_modify_attrs {
MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
};
enum mlx5_ib_devx_obj_query_attrs {
MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
};
enum mlx5_ib_devx_obj_query_async_attrs {
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_CMD_IN,
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_FD,
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_WR_ID,
MLX5_IB_ATTR_DEVX_OBJ_QUERY_ASYNC_OUT_LEN,
};
enum mlx5_ib_devx_subscribe_event_attrs {
MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE,
MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST,
MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM,
MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE,
};
enum mlx5_ib_devx_query_eqn_attrs {
MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
};
enum mlx5_ib_devx_obj_methods {
MLX5_IB_METHOD_DEVX_OBJ_CREATE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
MLX5_IB_METHOD_DEVX_OBJ_QUERY,
MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY,
};
enum mlx5_ib_var_alloc_attrs {
MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
};
enum mlx5_ib_var_obj_destroy_attrs {
MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_var_obj_methods {
MLX5_IB_METHOD_VAR_OBJ_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_VAR_OBJ_DESTROY,
};
enum mlx5_ib_uar_alloc_attrs {
MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
};
enum mlx5_ib_uar_obj_destroy_attrs {
MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_uar_obj_methods {
MLX5_IB_METHOD_UAR_OBJ_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_UAR_OBJ_DESTROY,
};
enum mlx5_ib_devx_umem_reg_attrs {
MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
};
enum mlx5_ib_devx_umem_dereg_attrs {
MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_pp_obj_methods {
MLX5_IB_METHOD_PP_OBJ_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_PP_OBJ_DESTROY,
};
enum mlx5_ib_pp_alloc_attrs {
MLX5_IB_ATTR_PP_OBJ_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_PP_OBJ_ALLOC_CTX,
MLX5_IB_ATTR_PP_OBJ_ALLOC_FLAGS,
MLX5_IB_ATTR_PP_OBJ_ALLOC_INDEX,
};
enum mlx5_ib_pp_obj_destroy_attrs {
MLX5_IB_ATTR_PP_OBJ_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_devx_umem_methods {
MLX5_IB_METHOD_DEVX_UMEM_REG = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_DEVX_UMEM_DEREG,
};
enum mlx5_ib_devx_async_cmd_fd_alloc_attrs {
MLX5_IB_ATTR_DEVX_ASYNC_CMD_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_devx_async_event_fd_alloc_attrs {
MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS,
};
enum mlx5_ib_devx_async_cmd_fd_methods {
MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_devx_async_event_fd_methods {
MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_objects {
MLX5_IB_OBJECT_DEVX = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_OBJECT_DEVX_OBJ,
MLX5_IB_OBJECT_DEVX_UMEM,
MLX5_IB_OBJECT_FLOW_MATCHER,
MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD,
MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD,
MLX5_IB_OBJECT_VAR,
MLX5_IB_OBJECT_PP,
MLX5_IB_OBJECT_UAR,
};
enum mlx5_ib_flow_matcher_create_attrs {
MLX5_IB_ATTR_FLOW_MATCHER_CREATE_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_FLOW_MATCHER_MATCH_MASK,
MLX5_IB_ATTR_FLOW_MATCHER_FLOW_TYPE,
MLX5_IB_ATTR_FLOW_MATCHER_MATCH_CRITERIA,
MLX5_IB_ATTR_FLOW_MATCHER_FLOW_FLAGS,
MLX5_IB_ATTR_FLOW_MATCHER_FT_TYPE,
};
enum mlx5_ib_flow_matcher_destroy_attrs {
MLX5_IB_ATTR_FLOW_MATCHER_DESTROY_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_flow_matcher_methods {
MLX5_IB_METHOD_FLOW_MATCHER_CREATE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_FLOW_MATCHER_DESTROY,
};
#define MLX5_IB_DW_MATCH_PARAM 0x80
struct mlx5_ib_match_params {
__u32 match_params[MLX5_IB_DW_MATCH_PARAM];
};
enum mlx5_ib_flow_type {
MLX5_IB_FLOW_TYPE_NORMAL,
MLX5_IB_FLOW_TYPE_SNIFFER,
MLX5_IB_FLOW_TYPE_ALL_DEFAULT,
MLX5_IB_FLOW_TYPE_MC_DEFAULT,
};
enum mlx5_ib_create_flow_attrs {
MLX5_IB_ATTR_CREATE_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_CREATE_FLOW_MATCH_VALUE,
MLX5_IB_ATTR_CREATE_FLOW_DEST_QP,
MLX5_IB_ATTR_CREATE_FLOW_DEST_DEVX,
MLX5_IB_ATTR_CREATE_FLOW_MATCHER,
MLX5_IB_ATTR_CREATE_FLOW_ARR_FLOW_ACTIONS,
MLX5_IB_ATTR_CREATE_FLOW_TAG,
MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX,
MLX5_IB_ATTR_CREATE_FLOW_ARR_COUNTERS_DEVX_OFFSET,
};
enum mlx5_ib_destoy_flow_attrs {
MLX5_IB_ATTR_DESTROY_FLOW_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
};
enum mlx5_ib_flow_methods {
MLX5_IB_METHOD_CREATE_FLOW = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_DESTROY_FLOW,
};
enum mlx5_ib_flow_action_methods {
MLX5_IB_METHOD_FLOW_ACTION_CREATE_MODIFY_HEADER = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_METHOD_FLOW_ACTION_CREATE_PACKET_REFORMAT,
};
enum mlx5_ib_create_flow_action_create_modify_header_attrs {
MLX5_IB_ATTR_CREATE_MODIFY_HEADER_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_CREATE_MODIFY_HEADER_ACTIONS_PRM,
MLX5_IB_ATTR_CREATE_MODIFY_HEADER_FT_TYPE,
};
enum mlx5_ib_create_flow_action_create_packet_reformat_attrs {
MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_HANDLE = (1U << UVERBS_ID_NS_SHIFT),
MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_TYPE,
MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_FT_TYPE,
MLX5_IB_ATTR_CREATE_PACKET_REFORMAT_DATA_BUF,
};
#endif

87
sys/ofed/include/uapi/rdma/mlx5_user_ioctl_verbs.h Normal file
View File

@ -0,0 +1,87 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef MLX5_USER_IOCTL_VERBS_H
#define MLX5_USER_IOCTL_VERBS_H
#include <linux/types.h>
enum mlx5_ib_uapi_flow_action_flags {
MLX5_IB_UAPI_FLOW_ACTION_FLAGS_REQUIRE_METADATA = 1 << 0,
};
enum mlx5_ib_uapi_flow_table_type {
MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_RX = 0x0,
MLX5_IB_UAPI_FLOW_TABLE_TYPE_NIC_TX = 0x1,
MLX5_IB_UAPI_FLOW_TABLE_TYPE_FDB = 0x2,
MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_RX = 0x3,
MLX5_IB_UAPI_FLOW_TABLE_TYPE_RDMA_TX = 0x4,
};
enum mlx5_ib_uapi_flow_action_packet_reformat_type {
MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TUNNEL_TO_L2 = 0x0,
MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L2_TUNNEL = 0x1,
MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L3_TUNNEL_TO_L2 = 0x2,
MLX5_IB_UAPI_FLOW_ACTION_PACKET_REFORMAT_TYPE_L2_TO_L3_TUNNEL = 0x3,
};
struct mlx5_ib_uapi_devx_async_cmd_hdr {
__aligned_u64 wr_id;
__u8 out_data[];
};
enum mlx5_ib_uapi_dm_type {
MLX5_IB_UAPI_DM_TYPE_MEMIC,
MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM,
MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM,
};
enum mlx5_ib_uapi_devx_create_event_channel_flags {
MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA = 1 << 0,
};
struct mlx5_ib_uapi_devx_async_event_hdr {
__aligned_u64 cookie;
__u8 out_data[];
};
enum mlx5_ib_uapi_pp_alloc_flags {
MLX5_IB_UAPI_PP_ALLOC_FLAGS_DEDICATED_INDEX = 1 << 0,
};
enum mlx5_ib_uapi_uar_alloc_type {
MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF = 0x0,
MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC = 0x1,
};
#endif

63
sys/ofed/include/uapi/rdma/rdma_user_ioctl.h Normal file
View File

@ -0,0 +1,63 @@
/* SPDX-License-Identifier: ((GPL-2.0 WITH Linux-syscall-note) OR Linux-OpenIB) */
/*
* Copyright (c) 2016 Mellanox Technologies, LTD. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef RDMA_USER_IOCTL_H
#define RDMA_USER_IOCTL_H
#ifdef _KERNEL
#include <linux/types.h>
#include <linux/ioctl.h>
#else
#include <infiniband/types.h>
#include <sys/ioccom.h>
#endif
#include <rdma/ib_user_mad.h>
#include <rdma/rdma_user_ioctl_cmds.h>
/* Legacy name, for user space application which already use it */
#define IB_IOCTL_MAGIC RDMA_IOCTL_MAGIC
/*
* General blocks assignments
* It is closed on purpose do not expose it it user space
* #define MAD_CMD_BASE 0x00
*/
/* MAD specific section */
#define IB_USER_MAD_REGISTER_AGENT _IOWR(RDMA_IOCTL_MAGIC, 0x01, struct ib_user_mad_reg_req)
#define IB_USER_MAD_UNREGISTER_AGENT _IOW(RDMA_IOCTL_MAGIC, 0x02, __u32)
#define IB_USER_MAD_ENABLE_PKEY _IO(RDMA_IOCTL_MAGIC, 0x03)
#define IB_USER_MAD_REGISTER_AGENT2 _IOWR(RDMA_IOCTL_MAGIC, 0x04, struct ib_user_mad_reg_req2)
#endif /* RDMA_USER_IOCTL_H */

92
sys/ofed/include/uapi/rdma/rdma_user_ioctl_cmds.h Normal file
View File

@ -0,0 +1,92 @@
/*
* Copyright (c) 2018, Mellanox Technologies inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* OpenIB.org BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#ifndef RDMA_USER_IOCTL_CMDS_H
#define RDMA_USER_IOCTL_CMDS_H
#ifdef _KERNEL
#include <linux/types.h>
#include <linux/ioctl.h>
#else
#include <infiniband/types.h>
#include <sys/ioccom.h>
#endif
/* Documentation/ioctl/ioctl-number.rst */
#define RDMA_IOCTL_MAGIC 0x1b
#define RDMA_VERBS_IOCTL \
_IOWR(RDMA_IOCTL_MAGIC, 1, struct ib_uverbs_ioctl_hdr)
enum {
/* User input */
UVERBS_ATTR_F_MANDATORY = 1U << 0,
/*
* Valid output bit should be ignored and considered set in
* mandatory fields. This bit is kernel output.
*/
UVERBS_ATTR_F_VALID_OUTPUT = 1U << 1,
};
struct ib_uverbs_attr {
__u16 attr_id; /* command specific type attribute */
__u16 len; /* only for pointers and IDRs array */
__u16 flags; /* combination of UVERBS_ATTR_F_XXXX */
union {
struct {
__u8 elem_id;
__u8 reserved;
} enum_data;
__u16 reserved;
} attr_data;
union {
/*
* ptr to command, inline data, idr/fd or
* ptr to __u32 array of IDRs
*/
__aligned_u64 data;
/* Used by FD_IN and FD_OUT */
__s64 data_s64;
};
};
struct ib_uverbs_ioctl_hdr {
__u16 length;
__u16 object_id;
__u16 method_id;
__u16 num_attrs;
__aligned_u64 reserved1;
__u32 driver_id;
__u32 reserved2;
struct ib_uverbs_attr attrs[0];
};
#endif