From 8dee9a7a44d02985810090df1d5c86af31d7186b Mon Sep 17 00:00:00 2001 From: Hans Petter Selasky Date: Mon, 13 Nov 2017 10:59:38 +0000 Subject: [PATCH] Remove no longer supported mthca driver. Sponsored by: Mellanox Technologies --- share/mk/bsd.libnames.mk | 1 - share/mk/src.libnames.mk | 3 - sys/amd64/conf/NOTES | 2 - sys/conf/files | 37 - sys/i386/conf/NOTES | 2 - sys/modules/Makefile | 4 - sys/modules/mthca/Makefile | 17 - sys/ofed/drivers/infiniband/Kconfig | 66 - sys/ofed/drivers/infiniband/Makefile | 17 - sys/ofed/drivers/infiniband/hw/mthca/Kconfig | 17 - sys/ofed/drivers/infiniband/hw/mthca/Makefile | 7 - .../infiniband/hw/mthca/mthca_allocator.c | 300 --- .../drivers/infiniband/hw/mthca/mthca_av.c | 374 --- .../drivers/infiniband/hw/mthca/mthca_catas.c | 200 -- .../drivers/infiniband/hw/mthca/mthca_cmd.c | 1940 -------------- .../drivers/infiniband/hw/mthca/mthca_cmd.h | 341 --- .../infiniband/hw/mthca/mthca_config_reg.h | 50 - .../drivers/infiniband/hw/mthca/mthca_cq.c | 992 ------- .../drivers/infiniband/hw/mthca/mthca_dev.h | 597 ----- .../infiniband/hw/mthca/mthca_doorbell.h | 109 - .../drivers/infiniband/hw/mthca/mthca_eq.c | 920 ------- .../drivers/infiniband/hw/mthca/mthca_mad.c | 346 --- .../drivers/infiniband/hw/mthca/mthca_main.c | 1363 ---------- .../drivers/infiniband/hw/mthca/mthca_mcg.c | 372 --- .../infiniband/hw/mthca/mthca_memfree.c | 881 ------- .../infiniband/hw/mthca/mthca_memfree.h | 179 -- .../drivers/infiniband/hw/mthca/mthca_mr.c | 985 ------- .../drivers/infiniband/hw/mthca/mthca_pd.c | 81 - .../infiniband/hw/mthca/mthca_profile.c | 285 -- .../infiniband/hw/mthca/mthca_profile.h | 59 - .../infiniband/hw/mthca/mthca_provider.c | 1448 ---------- .../infiniband/hw/mthca/mthca_provider.h | 343 --- .../drivers/infiniband/hw/mthca/mthca_qp.c | 2333 ----------------- .../drivers/infiniband/hw/mthca/mthca_reset.c | 303 --- .../drivers/infiniband/hw/mthca/mthca_srq.c | 715 ----- .../drivers/infiniband/hw/mthca/mthca_uar.c | 78 - .../drivers/infiniband/hw/mthca/mthca_user.h | 112 - .../drivers/infiniband/hw/mthca/mthca_wqe.h | 131 - sys/ofed/drivers/infiniband/util/Kconfig | 6 - targets/pseudo/userland/lib/Makefile.depend | 1 - 40 files changed, 16017 deletions(-) delete mode 100644 sys/modules/mthca/Makefile delete mode 100644 sys/ofed/drivers/infiniband/Kconfig delete mode 100644 sys/ofed/drivers/infiniband/Makefile delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/Kconfig delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/Makefile delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_config_reg.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_cq.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_doorbell.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_mad.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_mcg.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_pd.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_qp.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_srq.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_uar.c delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_user.h delete mode 100644 sys/ofed/drivers/infiniband/hw/mthca/mthca_wqe.h delete mode 100644 sys/ofed/drivers/infiniband/util/Kconfig diff --git a/share/mk/bsd.libnames.mk b/share/mk/bsd.libnames.mk index b95ea0937697..003edfa5f0eb 100644 --- a/share/mk/bsd.libnames.mk +++ b/share/mk/bsd.libnames.mk @@ -109,7 +109,6 @@ LIBMILTER?= ${LIBDESTDIR}${LIBDIR_BASE}/libmilter.a LIBMLX4?= ${LIBDESTDIR}${LIBDIR_BASE}/libmlx4.a LIBMP?= ${LIBDESTDIR}${LIBDIR_BASE}/libmp.a LIBMT?= ${LIBDESTDIR}${LIBDIR_BASE}/libmt.a -LIBMTHCA?= ${LIBDESTDIR}${LIBDIR_BASE}/libmthca.a LIBNANDFS?= ${LIBDESTDIR}${LIBDIR_BASE}/libnandfs.a LIBNCURSES?= ${LIBDESTDIR}${LIBDIR_BASE}/libncurses.a LIBNCURSESW?= ${LIBDESTDIR}${LIBDIR_BASE}/libncursesw.a diff --git a/share/mk/src.libnames.mk b/share/mk/src.libnames.mk index 703adb03cc56..2639f6bdf253 100644 --- a/share/mk/src.libnames.mk +++ b/share/mk/src.libnames.mk @@ -199,7 +199,6 @@ _LIBRARIES+= \ ibumad \ ibverbs \ mlx4 \ - mthca \ opensm \ osmcomp \ osmvendor \ @@ -336,7 +335,6 @@ _DP_ibcm= ibverbs _DP_ibmad= ibcommon ibumad _DP_ibumad= ibcommon _DP_mlx4= ibverbs pthread -_DP_mthca= ibverbs pthread _DP_opensm= pthread _DP_osmcomp= pthread _DP_osmvendor= ibumad opensm osmcomp pthread @@ -488,7 +486,6 @@ LIBIBMADDIR= ${OBJTOP}/contrib/ofed/usr.lib/libibmad LIBIBUMADDIR= ${OBJTOP}/contrib/ofed/usr.lib/libibumad LIBIBVERBSDIR= ${OBJTOP}/contrib/ofed/usr.lib/libibverbs LIBMLX4DIR= ${OBJTOP}/contrib/ofed/usr.lib/libmlx4 -LIBMTHCADIR= ${OBJTOP}/contrib/ofed/usr.lib/libmthca LIBOPENSMDIR= ${OBJTOP}/contrib/ofed/usr.lib/libopensm LIBOSMCOMPDIR= ${OBJTOP}/contrib/ofed/usr.lib/libosmcomp LIBOSMVENDORDIR= ${OBJTOP}/contrib/ofed/usr.lib/libosmvendor diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES index bd4bd7c9e60d..687aa4e173cd 100644 --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -317,7 +317,6 @@ options DRM_DEBUG # Include debug printfs (slow) # ixlv: Intel XL710 40Gbe VF PCIE Ethernet # mlx4ib: Mellanox ConnectX HCA InfiniBand # mlx4en: Mellanox ConnectX HCA Ethernet -# mthca: Mellanox HCA InfiniBand # nfe: nVidia nForce MCP on-board Ethernet Networking (BSD open source) # sfxge: Solarflare SFC9000 family 10Gb Ethernet adapters # vmx: VMware VMXNET3 Ethernet (BSD open source) @@ -338,7 +337,6 @@ device ixlv # Intel XL710 40Gbe VF PCIE Ethernet device mlx4 # Shared code module between IB and Ethernet device mlx4ib # Mellanox ConnectX HCA InfiniBand device mlx4en # Mellanox ConnectX HCA Ethernet -device mthca # Mellanox HCA InfiniBand device nfe # nVidia nForce MCP on-board Ethernet device sfxge # Solarflare SFC9000 10Gb Ethernet device vmx # VMware VMXNET3 Ethernet diff --git a/sys/conf/files b/sys/conf/files index 09dee9979315..a79619d650e3 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -4696,43 +4696,6 @@ dev/mlx5/mlx5_en/mlx5_en_rx.c optional mlx5en pci inet inet6 \ dev/mlx5/mlx5_en/mlx5_en_txrx.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_allocator.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_av.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_catas.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_cmd.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_cq.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_eq.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_mad.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_main.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_mcg.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_memfree.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_mr.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_pd.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_profile.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_provider.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_qp.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_reset.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_srq.c optional mthca \ - compile-with "${OFED_C}" -ofed/drivers/infiniband/hw/mthca/mthca_uar.c optional mthca \ - compile-with "${OFED_C}" - # crypto support opencrypto/cast.c optional crypto | ipsec | ipsec_support opencrypto/criov.c optional crypto | ipsec | ipsec_support diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index 20a1c7db3789..648e8a42400f 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -555,7 +555,6 @@ hint.mse.0.irq="5" # Requires the iwn firmware module # mlx4ib: Mellanox ConnectX HCA InfiniBand # mlx4en: Mellanox ConnectX HCA Ethernet -# mthca: Mellanox HCA InfiniBand # nfe: nVidia nForce MCP on-board Ethernet Networking (BSD open source) # sbni: Granch SBNI12-xx ISA and PCI adapters # vmx: VMware VMXNET3 Ethernet (BSD open source) @@ -596,7 +595,6 @@ hint.le.0.drq="0" device mlx4 # Shared code module between IB and Ethernet device mlx4ib # Mellanox ConnectX HCA InfiniBand device mlx4en # Mellanox ConnectX HCA Ethernet -device mthca # Mellanox HCA InfiniBand device nfe # nVidia nForce MCP on-board Ethernet device sbni hint.sbni.0.at="isa" diff --git a/sys/modules/Makefile b/sys/modules/Makefile index 7d90d5ab444e..c04d970e32ae 100644 --- a/sys/modules/Makefile +++ b/sys/modules/Makefile @@ -264,7 +264,6 @@ SUBDIR= \ msdosfs_iconv \ ${_mse} \ msk \ - ${_mthca} \ mvs \ mwl \ ${_mwlfw} \ @@ -680,9 +679,6 @@ _mlx4ib= mlx4ib _mlx5ib= mlx5ib .endif _mly= mly -.if ${MK_OFED} != "no" || defined(ALL_MODULES) -_mthca= mthca -.endif _nfe= nfe _nvd= nvd _nvme= nvme diff --git a/sys/modules/mthca/Makefile b/sys/modules/mthca/Makefile deleted file mode 100644 index 2c057f254f8d..000000000000 --- a/sys/modules/mthca/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# $FreeBSD$ -.PATH: ${SRCTOP}/sys/ofed/drivers/infiniband/hw/mthca - -KMOD = mthca -SRCS = device_if.h bus_if.h pci_if.h vnode_if.h -SRCS+= mthca_allocator.c mthca_av.c mthca_catas.c mthca_cmd.c mthca_cq.c -SRCS+= mthca_eq.c mthca_mad.c mthca_main.c mthca_mcg.c mthca_memfree.c -SRCS+= mthca_mr.c mthca_pd.c mthca_profile.c mthca_provider.c mthca_qp.c -SRCS+= mthca_reset.c mthca_srq.c mthca_uar.c -SRCS+= opt_inet.h opt_inet6.h - -CFLAGS+= -I${SRCTOP}/sys/ofed/include -CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include - -.include - -CFLAGS+= -Wno-cast-qual -Wno-pointer-arith diff --git a/sys/ofed/drivers/infiniband/Kconfig b/sys/ofed/drivers/infiniband/Kconfig deleted file mode 100644 index 0a2ef113622c..000000000000 --- a/sys/ofed/drivers/infiniband/Kconfig +++ /dev/null @@ -1,66 +0,0 @@ -menuconfig INFINIBAND - tristate "InfiniBand support" - depends on PCI || BROKEN - depends on HAS_IOMEM - ---help--- - Core support for InfiniBand (IB). Make sure to also select - any protocols you wish to use as well as drivers for your - InfiniBand hardware. - -if INFINIBAND - -config INFINIBAND_USER_MAD - tristate "InfiniBand userspace MAD support" - depends on INFINIBAND - ---help--- - Userspace InfiniBand Management Datagram (MAD) support. This - is the kernel side of the userspace MAD support, which allows - userspace processes to send and receive MADs. You will also - need libibumad from . - -config INFINIBAND_USER_ACCESS - tristate "InfiniBand userspace access (verbs and CM)" - ---help--- - Userspace InfiniBand access support. This enables the - kernel side of userspace verbs and the userspace - communication manager (CM). This allows userspace processes - to set up connections and directly access InfiniBand - hardware for fast-path operations. You will also need - libibverbs, libibcm and a hardware driver library from - . - -config INFINIBAND_USER_MEM - bool - depends on INFINIBAND_USER_ACCESS != n - default y - -config INFINIBAND_ADDR_TRANS - bool - depends on INET - depends on !(INFINIBAND = y && IPV6 = m) - default y - -source "drivers/infiniband/hw/mthca/Kconfig" -source "drivers/infiniband/hw/ipath/Kconfig" -source "drivers/infiniband/hw/qib/Kconfig" -source "drivers/infiniband/hw/ehca/Kconfig" -source "drivers/infiniband/hw/amso1100/Kconfig" -source "drivers/infiniband/hw/cxgb3/Kconfig" -source "drivers/infiniband/hw/mlx4/Kconfig" -source "drivers/infiniband/hw/nes/Kconfig" - -source "drivers/infiniband/ulp/ipoib/Kconfig" - -source "drivers/infiniband/ulp/srp/Kconfig" - -source "drivers/infiniband/ulp/srpt/Kconfig" - -source "drivers/infiniband/ulp/iser/Kconfig" - -source "drivers/infiniband/ulp/sdp/Kconfig" - -source "drivers/infiniband/ulp/qlgc_vnic/Kconfig" - -source "drivers/infiniband/util/Kconfig" - -endif # INFINIBAND diff --git a/sys/ofed/drivers/infiniband/Makefile b/sys/ofed/drivers/infiniband/Makefile deleted file mode 100644 index ea5dbe0fda22..000000000000 --- a/sys/ofed/drivers/infiniband/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -obj-$(CONFIG_INFINIBAND) += core/ -obj-$(CONFIG_INFINIBAND_MTHCA) += hw/mthca/ -obj-$(CONFIG_INFINIBAND_IPATH) += hw/ipath/ -obj-$(CONFIG_INFINIBAND_QIB) += hw/qib/ -obj-$(CONFIG_INFINIBAND_EHCA) += hw/ehca/ -obj-$(CONFIG_INFINIBAND_AMSO1100) += hw/amso1100/ -obj-$(CONFIG_INFINIBAND_CXGB3) += hw/cxgb3/ -obj-$(CONFIG_INFINIBAND_NES) += hw/nes/ -obj-$(CONFIG_MLX4_INFINIBAND) += hw/mlx4/ -obj-$(CONFIG_INFINIBAND_NES) += hw/nes/ -obj-$(CONFIG_INFINIBAND_IPOIB) += ulp/ipoib/ -obj-$(CONFIG_INFINIBAND_SRP) += ulp/srp/ -obj-$(CONFIG_INFINIBAND_SRPT) += ulp/srpt/ -obj-$(CONFIG_INFINIBAND_ISER) += ulp/iser/ -obj-$(CONFIG_INFINIBAND_SDP) += ulp/sdp/ -obj-$(CONFIG_INFINIBAND_QLGC_VNIC) += ulp/qlgc_vnic/ -obj-$(CONFIG_INFINIBAND_MADEYE) += util/ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/Kconfig b/sys/ofed/drivers/infiniband/hw/mthca/Kconfig deleted file mode 100644 index 03efc074967e..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/Kconfig +++ /dev/null @@ -1,17 +0,0 @@ -config INFINIBAND_MTHCA - tristate "Mellanox HCA support" - depends on PCI - ---help--- - This is a low-level driver for Mellanox InfiniHost host - channel adapters (HCAs), including the MT23108 PCI-X HCA - ("Tavor") and the MT25208 PCI Express HCA ("Arbel"). - -config INFINIBAND_MTHCA_DEBUG - bool "Verbose debugging output" if EMBEDDED - depends on INFINIBAND_MTHCA - default y - ---help--- - This option causes debugging code to be compiled into the - mthca driver. The output can be turned on via the - debug_level module parameter (which can also be set after - the driver is loaded through sysfs). diff --git a/sys/ofed/drivers/infiniband/hw/mthca/Makefile b/sys/ofed/drivers/infiniband/hw/mthca/Makefile deleted file mode 100644 index e388d95d0cf1..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -obj-$(CONFIG_INFINIBAND_MTHCA) += ib_mthca.o - -ib_mthca-y := mthca_main.o mthca_cmd.o mthca_profile.o mthca_reset.o \ - mthca_allocator.o mthca_eq.o mthca_pd.o mthca_cq.o \ - mthca_mr.o mthca_qp.o mthca_av.o mthca_mcg.o mthca_mad.o \ - mthca_provider.o mthca_memfree.o mthca_uar.o mthca_srq.o \ - mthca_catas.o diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c deleted file mode 100644 index 8e9b0185cb28..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_allocator.c +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include "mthca_dev.h" - -/* Trivial bitmap-based allocator */ -u32 mthca_alloc(struct mthca_alloc *alloc) -{ - unsigned long flags; - u32 obj; - - spin_lock_irqsave(&alloc->lock, flags); - - obj = find_next_zero_bit(alloc->table, alloc->max, alloc->last); - if (obj >= alloc->max) { - alloc->top = (alloc->top + alloc->max) & alloc->mask; - obj = find_first_zero_bit(alloc->table, alloc->max); - } - - if (obj < alloc->max) { - set_bit(obj, alloc->table); - obj |= alloc->top; - } else - obj = -1; - - spin_unlock_irqrestore(&alloc->lock, flags); - - return obj; -} - -void mthca_free(struct mthca_alloc *alloc, u32 obj) -{ - unsigned long flags; - - obj &= alloc->max - 1; - - spin_lock_irqsave(&alloc->lock, flags); - - clear_bit(obj, alloc->table); - alloc->last = min(alloc->last, obj); - alloc->top = (alloc->top + alloc->max) & alloc->mask; - - spin_unlock_irqrestore(&alloc->lock, flags); -} - -int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, - u32 reserved) -{ - int i; - - /* num must be a power of 2 */ - if (num != 1 << (ffs(num) - 1)) - return -EINVAL; - - alloc->last = 0; - alloc->top = 0; - alloc->max = num; - alloc->mask = mask; - spin_lock_init(&alloc->lock); - alloc->table = kmalloc(BITS_TO_LONGS(num) * sizeof (long), - GFP_KERNEL); - if (!alloc->table) - return -ENOMEM; - - bitmap_zero(alloc->table, num); - for (i = 0; i < reserved; ++i) - set_bit(i, alloc->table); - - return 0; -} - -void mthca_alloc_cleanup(struct mthca_alloc *alloc) -{ - kfree(alloc->table); -} - -/* - * Array of pointers with lazy allocation of leaf pages. Callers of - * _get, _set and _clear methods must use a lock or otherwise - * serialize access to the array. - */ - -#define MTHCA_ARRAY_MASK (PAGE_SIZE / sizeof (void *) - 1) - -void *mthca_array_get(struct mthca_array *array, int index) -{ - int p = (index * sizeof (void *)) >> PAGE_SHIFT; - - if (array->page_list[p].page) - return array->page_list[p].page[index & MTHCA_ARRAY_MASK]; - else - return NULL; -} - -int mthca_array_set(struct mthca_array *array, int index, void *value) -{ - int p = (index * sizeof (void *)) >> PAGE_SHIFT; - - /* Allocate with GFP_ATOMIC because we'll be called with locks held. */ - if (!array->page_list[p].page) - array->page_list[p].page = (void **) get_zeroed_page(GFP_ATOMIC); - - if (!array->page_list[p].page) - return -ENOMEM; - - array->page_list[p].page[index & MTHCA_ARRAY_MASK] = value; - ++array->page_list[p].used; - - return 0; -} - -void mthca_array_clear(struct mthca_array *array, int index) -{ - int p = (index * sizeof (void *)) >> PAGE_SHIFT; - - if (--array->page_list[p].used == 0) { - free_page((unsigned long) array->page_list[p].page); - array->page_list[p].page = NULL; - } else - array->page_list[p].page[index & MTHCA_ARRAY_MASK] = NULL; - - if (array->page_list[p].used < 0) - pr_debug("Array %p index %d page %d with ref count %d < 0\n", - array, index, p, array->page_list[p].used); -} - -int mthca_array_init(struct mthca_array *array, int nent) -{ - int npage = (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; - int i; - - array->page_list = kmalloc(npage * sizeof *array->page_list, GFP_KERNEL); - if (!array->page_list) - return -ENOMEM; - - for (i = 0; i < npage; ++i) { - array->page_list[i].page = NULL; - array->page_list[i].used = 0; - } - - return 0; -} - -void mthca_array_cleanup(struct mthca_array *array, int nent) -{ - int i; - - for (i = 0; i < (nent * sizeof (void *) + PAGE_SIZE - 1) / PAGE_SIZE; ++i) - free_page((unsigned long) array->page_list[i].page); - - kfree(array->page_list); -} - -/* - * Handling for queue buffers -- we allocate a bunch of memory and - * register it in a memory region at HCA virtual address 0. If the - * requested size is > max_direct, we split the allocation into - * multiple pages, so we don't require too much contiguous memory. - */ - -int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, - union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, - int hca_write, struct mthca_mr *mr) -{ - int err = -ENOMEM; - int npages, shift; - u64 *dma_list = NULL; - dma_addr_t t; - int i; - - if (size <= max_direct) { - *is_direct = 1; - npages = 1; - shift = get_order(size) + PAGE_SHIFT; - - buf->direct.buf = dma_alloc_coherent(&dev->pdev->dev, - size, &t, GFP_KERNEL); - if (!buf->direct.buf) - return -ENOMEM; - - pci_unmap_addr_set(&buf->direct, mapping, t); - - memset(buf->direct.buf, 0, size); - - while (t & ((1 << shift) - 1)) { - --shift; - npages *= 2; - } - - dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); - if (!dma_list) - goto err_free; - - for (i = 0; i < npages; ++i) - dma_list[i] = t + i * (1 << shift); - } else { - *is_direct = 0; - npages = (size + PAGE_SIZE - 1) / PAGE_SIZE; - shift = PAGE_SHIFT; - - dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); - if (!dma_list) - return -ENOMEM; - - buf->page_list = kmalloc(npages * sizeof *buf->page_list, - GFP_KERNEL); - if (!buf->page_list) - goto err_out; - - for (i = 0; i < npages; ++i) - buf->page_list[i].buf = NULL; - - for (i = 0; i < npages; ++i) { - buf->page_list[i].buf = - dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE, - &t, GFP_KERNEL); - if (!buf->page_list[i].buf) - goto err_free; - - dma_list[i] = t; - pci_unmap_addr_set(&buf->page_list[i], mapping, t); - - clear_page(buf->page_list[i].buf); - } - } - - err = mthca_mr_alloc_phys(dev, pd->pd_num, - dma_list, shift, npages, - 0, size, - MTHCA_MPT_FLAG_LOCAL_READ | - (hca_write ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0), - mr); - if (err) - goto err_free; - - kfree(dma_list); - - return 0; - -err_free: - mthca_buf_free(dev, size, buf, *is_direct, NULL); - -err_out: - kfree(dma_list); - - return err; -} - -void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, - int is_direct, struct mthca_mr *mr) -{ - int i; - - if (mr) - mthca_free_mr(dev, mr); - - if (is_direct) - dma_free_coherent(&dev->pdev->dev, size, buf->direct.buf, - pci_unmap_addr(&buf->direct, mapping)); - else { - for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, - buf->page_list[i].buf, - pci_unmap_addr(&buf->page_list[i], - mapping)); - kfree(buf->page_list); - } -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c deleted file mode 100644 index 32f6c6315454..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_av.c +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include -#include - -#include "mthca_dev.h" - -enum { - MTHCA_RATE_TAVOR_FULL = 0, - MTHCA_RATE_TAVOR_1X = 1, - MTHCA_RATE_TAVOR_4X = 2, - MTHCA_RATE_TAVOR_1X_DDR = 3 -}; - -enum { - MTHCA_RATE_MEMFREE_FULL = 0, - MTHCA_RATE_MEMFREE_QUARTER = 1, - MTHCA_RATE_MEMFREE_EIGHTH = 2, - MTHCA_RATE_MEMFREE_HALF = 3 -}; - -struct mthca_av { - __be32 port_pd; - u8 reserved1; - u8 g_slid; - __be16 dlid; - u8 reserved2; - u8 gid_index; - u8 msg_sr; - u8 hop_limit; - __be32 sl_tclass_flowlabel; - __be32 dgid[4]; -}; - -static enum ib_rate memfree_rate_to_ib(u8 mthca_rate, u8 port_rate) -{ - switch (mthca_rate) { - case MTHCA_RATE_MEMFREE_EIGHTH: - return mult_to_ib_rate(port_rate >> 3); - case MTHCA_RATE_MEMFREE_QUARTER: - return mult_to_ib_rate(port_rate >> 2); - case MTHCA_RATE_MEMFREE_HALF: - return mult_to_ib_rate(port_rate >> 1); - case MTHCA_RATE_MEMFREE_FULL: - default: - return mult_to_ib_rate(port_rate); - } -} - -static enum ib_rate tavor_rate_to_ib(u8 mthca_rate, u8 port_rate) -{ - switch (mthca_rate) { - case MTHCA_RATE_TAVOR_1X: return IB_RATE_2_5_GBPS; - case MTHCA_RATE_TAVOR_1X_DDR: return IB_RATE_5_GBPS; - case MTHCA_RATE_TAVOR_4X: return IB_RATE_10_GBPS; - default: return mult_to_ib_rate(port_rate); - } -} - -enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u8 port) -{ - if (mthca_is_memfree(dev)) { - /* Handle old Arbel FW */ - if (dev->limits.stat_rate_support == 0x3 && mthca_rate) - return IB_RATE_2_5_GBPS; - - return memfree_rate_to_ib(mthca_rate, dev->rate[port - 1]); - } else - return tavor_rate_to_ib(mthca_rate, dev->rate[port - 1]); -} - -static u8 ib_rate_to_memfree(u8 req_rate, u8 cur_rate) -{ - if (cur_rate <= req_rate) - return 0; - - /* - * Inter-packet delay (IPD) to get from rate X down to a rate - * no more than Y is (X - 1) / Y. - */ - switch ((cur_rate - 1) / req_rate) { - case 0: return MTHCA_RATE_MEMFREE_FULL; - case 1: return MTHCA_RATE_MEMFREE_HALF; - case 2: /* fall through */ - case 3: return MTHCA_RATE_MEMFREE_QUARTER; - default: return MTHCA_RATE_MEMFREE_EIGHTH; - } -} - -static u8 ib_rate_to_tavor(u8 static_rate) -{ - switch (static_rate) { - case IB_RATE_2_5_GBPS: return MTHCA_RATE_TAVOR_1X; - case IB_RATE_5_GBPS: return MTHCA_RATE_TAVOR_1X_DDR; - case IB_RATE_10_GBPS: return MTHCA_RATE_TAVOR_4X; - default: return MTHCA_RATE_TAVOR_FULL; - } -} - -u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u8 port) -{ - u8 rate; - - if (!static_rate || ib_rate_to_mult(static_rate) >= dev->rate[port - 1]) - return 0; - - if (mthca_is_memfree(dev)) - rate = ib_rate_to_memfree(ib_rate_to_mult(static_rate), - dev->rate[port - 1]); - else - rate = ib_rate_to_tavor(static_rate); - - if (!(dev->limits.stat_rate_support & (1 << rate))) - rate = 1; - - return rate; -} - -int mthca_create_ah(struct mthca_dev *dev, - struct mthca_pd *pd, - struct ib_ah_attr *ah_attr, - struct mthca_ah *ah) -{ - u32 index = -1; - struct mthca_av *av = NULL; - - ah->type = MTHCA_AH_PCI_POOL; - - if (mthca_is_memfree(dev)) { - ah->av = kmalloc(sizeof *ah->av, GFP_ATOMIC); - if (!ah->av) - return -ENOMEM; - - ah->type = MTHCA_AH_KMALLOC; - av = ah->av; - } else if (!atomic_read(&pd->sqp_count) && - !(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { - index = mthca_alloc(&dev->av_table.alloc); - - /* fall back to allocate in host memory */ - if (index == -1) - goto on_hca_fail; - - av = kmalloc(sizeof *av, GFP_ATOMIC); - if (!av) - goto on_hca_fail; - - ah->type = MTHCA_AH_ON_HCA; - ah->avdma = dev->av_table.ddr_av_base + - index * MTHCA_AV_SIZE; - } - -on_hca_fail: - if (ah->type == MTHCA_AH_PCI_POOL) { - ah->av = pci_pool_alloc(dev->av_table.pool, - GFP_ATOMIC, &ah->avdma); - if (!ah->av) - return -ENOMEM; - - av = ah->av; - } - - ah->key = pd->ntmr.ibmr.lkey; - - memset(av, 0, MTHCA_AV_SIZE); - - av->port_pd = cpu_to_be32(pd->pd_num | (ah_attr->port_num << 24)); - av->g_slid = ah_attr->src_path_bits; - av->dlid = cpu_to_be16(ah_attr->dlid); - av->msg_sr = (3 << 4) | /* 2K message */ - mthca_get_rate(dev, ah_attr->static_rate, ah_attr->port_num); - av->sl_tclass_flowlabel = cpu_to_be32(ah_attr->sl << 28); - if (ah_attr->ah_flags & IB_AH_GRH) { - av->g_slid |= 0x80; - av->gid_index = (ah_attr->port_num - 1) * dev->limits.gid_table_len + - ah_attr->grh.sgid_index; - av->hop_limit = ah_attr->grh.hop_limit; - av->sl_tclass_flowlabel |= - cpu_to_be32((ah_attr->grh.traffic_class << 20) | - ah_attr->grh.flow_label); - memcpy(av->dgid, ah_attr->grh.dgid.raw, 16); - } else { - /* Arbel workaround -- low byte of GID must be 2 */ - av->dgid[3] = cpu_to_be32(2); - } - - if (0) { - int j; - - mthca_dbg(dev, "Created UDAV at %p/%08lx:\n", - av, (unsigned long) ah->avdma); - for (j = 0; j < 8; ++j) - printk(KERN_DEBUG " [%2x] %08x\n", - j * 4, be32_to_cpu(((__be32 *) av)[j])); - } - - if (ah->type == MTHCA_AH_ON_HCA) { - memcpy_toio(dev->av_table.av_map + index * MTHCA_AV_SIZE, - av, MTHCA_AV_SIZE); - kfree(av); - } - - return 0; -} - -int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah) -{ - switch (ah->type) { - case MTHCA_AH_ON_HCA: - mthca_free(&dev->av_table.alloc, - (ah->avdma - dev->av_table.ddr_av_base) / - MTHCA_AV_SIZE); - break; - - case MTHCA_AH_PCI_POOL: - pci_pool_free(dev->av_table.pool, ah->av, ah->avdma); - break; - - case MTHCA_AH_KMALLOC: - kfree(ah->av); - break; - } - - return 0; -} - -int mthca_ah_grh_present(struct mthca_ah *ah) -{ - return !!(ah->av->g_slid & 0x80); -} - -int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah, - struct ib_ud_header *header) -{ - if (ah->type == MTHCA_AH_ON_HCA) - return -EINVAL; - - header->lrh.service_level = be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 28; - header->lrh.destination_lid = ah->av->dlid; - header->lrh.source_lid = cpu_to_be16(ah->av->g_slid & 0x7f); - if (mthca_ah_grh_present(ah)) { - header->grh.traffic_class = - (be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20) & 0xff; - header->grh.flow_label = - ah->av->sl_tclass_flowlabel & cpu_to_be32(0xfffff); - header->grh.hop_limit = ah->av->hop_limit; - ib_get_cached_gid(&dev->ib_dev, - be32_to_cpu(ah->av->port_pd) >> 24, - ah->av->gid_index % dev->limits.gid_table_len, - &header->grh.source_gid); - memcpy(header->grh.destination_gid.raw, - ah->av->dgid, 16); - } - - return 0; -} - -int mthca_ah_query(struct ib_ah *ibah, struct ib_ah_attr *attr) -{ - struct mthca_ah *ah = to_mah(ibah); - struct mthca_dev *dev = to_mdev(ibah->device); - - /* Only implement for MAD and memfree ah for now. */ - if (ah->type == MTHCA_AH_ON_HCA) - return -ENOSYS; - - memset(attr, 0, sizeof *attr); - attr->dlid = be16_to_cpu(ah->av->dlid); - attr->sl = be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 28; - attr->port_num = be32_to_cpu(ah->av->port_pd) >> 24; - attr->static_rate = mthca_rate_to_ib(dev, ah->av->msg_sr & 0x7, - attr->port_num); - attr->src_path_bits = ah->av->g_slid & 0x7F; - attr->ah_flags = mthca_ah_grh_present(ah) ? IB_AH_GRH : 0; - - if (attr->ah_flags) { - attr->grh.traffic_class = - be32_to_cpu(ah->av->sl_tclass_flowlabel) >> 20; - attr->grh.flow_label = - be32_to_cpu(ah->av->sl_tclass_flowlabel) & 0xfffff; - attr->grh.hop_limit = ah->av->hop_limit; - attr->grh.sgid_index = ah->av->gid_index & - (dev->limits.gid_table_len - 1); - memcpy(attr->grh.dgid.raw, ah->av->dgid, 16); - } - - return 0; -} - -int mthca_init_av_table(struct mthca_dev *dev) -{ - int err; - - if (mthca_is_memfree(dev)) - return 0; - - err = mthca_alloc_init(&dev->av_table.alloc, - dev->av_table.num_ddr_avs, - dev->av_table.num_ddr_avs - 1, - 0); - if (err) - return err; - - dev->av_table.pool = pci_pool_create("mthca_av", dev->pdev, - MTHCA_AV_SIZE, - MTHCA_AV_SIZE, 0); - if (!dev->av_table.pool) - goto out_free_alloc; - - if (!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { - dev->av_table.av_map = ioremap(pci_resource_start(dev->pdev, 4) + - dev->av_table.ddr_av_base - - dev->ddr_start, - dev->av_table.num_ddr_avs * - MTHCA_AV_SIZE); - if (!dev->av_table.av_map) - goto out_free_pool; - } else - dev->av_table.av_map = NULL; - - return 0; - - out_free_pool: - pci_pool_destroy(dev->av_table.pool); - - out_free_alloc: - mthca_alloc_cleanup(&dev->av_table.alloc); - return -ENOMEM; -} - -void mthca_cleanup_av_table(struct mthca_dev *dev) -{ - if (mthca_is_memfree(dev)) - return; - - if (dev->av_table.av_map) - iounmap(dev->av_table.av_map); - pci_pool_destroy(dev->av_table.pool); - mthca_alloc_cleanup(&dev->av_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c deleted file mode 100644 index 7f72140f227c..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_catas.c +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mthca_ - -#include -#include -#include - -#include "mthca_dev.h" - -enum { - MTHCA_CATAS_TYPE_INTERNAL = 0, - MTHCA_CATAS_TYPE_UPLINK = 3, - MTHCA_CATAS_TYPE_DDR = 4, - MTHCA_CATAS_TYPE_PARITY = 5, -}; - -#define MTHCA_CATAS_POLL_INTERVAL (5 * HZ) - -static DEFINE_SPINLOCK(catas_lock); - -static LIST_HEAD(catas_list); -static struct workqueue_struct *catas_wq; -static struct work_struct catas_work; - -static int catas_reset_disable; -module_param_named(catas_reset_disable, catas_reset_disable, int, 0644); -MODULE_PARM_DESC(catas_reset_disable, "disable reset on catastrophic event if nonzero"); - -static void catas_reset(struct work_struct *work) -{ - struct mthca_dev *dev, *tmpdev; - LIST_HEAD(tlist); - int ret; - - mutex_lock(&mthca_device_mutex); - - spin_lock_irq(&catas_lock); - list_splice_init(&catas_list, &tlist); - spin_unlock_irq(&catas_lock); - - list_for_each_entry_safe(dev, tmpdev, &tlist, catas_err.list) { - struct pci_dev *pdev = dev->pdev; - ret = __mthca_restart_one(dev->pdev); - /* 'dev' now is not valid */ - if (ret) - printk(KERN_ERR "mthca %s: Reset failed (%d)\n", - pci_name(pdev), ret); - else { - struct mthca_dev *d = pci_get_drvdata(pdev); - mthca_dbg(d, "Reset succeeded\n"); - } - } - - mutex_unlock(&mthca_device_mutex); -} - -static void handle_catas(struct mthca_dev *dev) -{ - struct ib_event event; - unsigned long flags; - const char *type; - int i; - - event.device = &dev->ib_dev; - event.event = IB_EVENT_DEVICE_FATAL; - event.element.port_num = 0; - dev->active = 0; - - ib_dispatch_event(&event); - - switch (swab32(readl(dev->catas_err.map)) >> 24) { - case MTHCA_CATAS_TYPE_INTERNAL: - type = "internal error"; - break; - case MTHCA_CATAS_TYPE_UPLINK: - type = "uplink bus error"; - break; - case MTHCA_CATAS_TYPE_DDR: - type = "DDR data error"; - break; - case MTHCA_CATAS_TYPE_PARITY: - type = "internal parity error"; - break; - default: - type = "unknown error"; - break; - } - - mthca_err(dev, "Catastrophic error detected: %s\n", type); - for (i = 0; i < dev->catas_err.size; ++i) - mthca_err(dev, " buf[%02x]: %08x\n", - i, swab32(readl(dev->catas_err.map + i))); - - if (catas_reset_disable) - return; - - spin_lock_irqsave(&catas_lock, flags); - list_add(&dev->catas_err.list, &catas_list); - queue_work(catas_wq, &catas_work); - spin_unlock_irqrestore(&catas_lock, flags); -} - -static void poll_catas(unsigned long dev_ptr) -{ - struct mthca_dev *dev = (struct mthca_dev *) dev_ptr; - int i; - - for (i = 0; i < dev->catas_err.size; ++i) - if (readl(dev->catas_err.map + i)) { - handle_catas(dev); - return; - } - - mod_timer(&dev->catas_err.timer, - round_jiffies(jiffies + MTHCA_CATAS_POLL_INTERVAL)); -} - -void mthca_start_catas_poll(struct mthca_dev *dev) -{ - unsigned long addr; - - init_timer(&dev->catas_err.timer); - dev->catas_err.map = NULL; - - addr = pci_resource_start(dev->pdev, 0) + - ((pci_resource_len(dev->pdev, 0) - 1) & - dev->catas_err.addr); - - dev->catas_err.map = ioremap(addr, dev->catas_err.size * 4); - if (!dev->catas_err.map) { - mthca_warn(dev, "couldn't map catastrophic error region " - "at 0x%lx/0x%x\n", addr, dev->catas_err.size * 4); - return; - } - - dev->catas_err.timer.data = (unsigned long) dev; - dev->catas_err.timer.function = poll_catas; - dev->catas_err.timer.expires = jiffies + MTHCA_CATAS_POLL_INTERVAL; - INIT_LIST_HEAD(&dev->catas_err.list); - add_timer(&dev->catas_err.timer); -} - -void mthca_stop_catas_poll(struct mthca_dev *dev) -{ - del_timer_sync(&dev->catas_err.timer); - - if (dev->catas_err.map) - iounmap(dev->catas_err.map); - - spin_lock_irq(&catas_lock); - list_del(&dev->catas_err.list); - spin_unlock_irq(&catas_lock); -} - -int __init mthca_catas_init(void) -{ - INIT_WORK(&catas_work, catas_reset); - - catas_wq = create_singlethread_workqueue("mthcacatas"); - if (!catas_wq) - return -ENOMEM; - - return 0; -} - -void mthca_catas_cleanup(void) -{ - destroy_workqueue(catas_wq); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c deleted file mode 100644 index 5c7b9ffd8905..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.c +++ /dev/null @@ -1,1940 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mthca_ - -#include -#include -#include -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_config_reg.h" -#include "mthca_cmd.h" -#include "mthca_memfree.h" - -#define CMD_POLL_TOKEN 0xffff - -enum { - HCR_IN_PARAM_OFFSET = 0x00, - HCR_IN_MODIFIER_OFFSET = 0x08, - HCR_OUT_PARAM_OFFSET = 0x0c, - HCR_TOKEN_OFFSET = 0x14, - HCR_STATUS_OFFSET = 0x18, - - HCR_OPMOD_SHIFT = 12, - HCA_E_BIT = 22, - HCR_GO_BIT = 23 -}; - -enum { - /* initialization and general commands */ - CMD_SYS_EN = 0x1, - CMD_SYS_DIS = 0x2, - CMD_MAP_FA = 0xfff, - CMD_UNMAP_FA = 0xffe, - CMD_RUN_FW = 0xff6, - CMD_MOD_STAT_CFG = 0x34, - CMD_QUERY_DEV_LIM = 0x3, - CMD_QUERY_FW = 0x4, - CMD_ENABLE_LAM = 0xff8, - CMD_DISABLE_LAM = 0xff7, - CMD_QUERY_DDR = 0x5, - CMD_QUERY_ADAPTER = 0x6, - CMD_INIT_HCA = 0x7, - CMD_CLOSE_HCA = 0x8, - CMD_INIT_IB = 0x9, - CMD_CLOSE_IB = 0xa, - CMD_QUERY_HCA = 0xb, - CMD_SET_IB = 0xc, - CMD_ACCESS_DDR = 0x2e, - CMD_MAP_ICM = 0xffa, - CMD_UNMAP_ICM = 0xff9, - CMD_MAP_ICM_AUX = 0xffc, - CMD_UNMAP_ICM_AUX = 0xffb, - CMD_SET_ICM_SIZE = 0xffd, - - /* TPT commands */ - CMD_SW2HW_MPT = 0xd, - CMD_QUERY_MPT = 0xe, - CMD_HW2SW_MPT = 0xf, - CMD_READ_MTT = 0x10, - CMD_WRITE_MTT = 0x11, - CMD_SYNC_TPT = 0x2f, - - /* EQ commands */ - CMD_MAP_EQ = 0x12, - CMD_SW2HW_EQ = 0x13, - CMD_HW2SW_EQ = 0x14, - CMD_QUERY_EQ = 0x15, - - /* CQ commands */ - CMD_SW2HW_CQ = 0x16, - CMD_HW2SW_CQ = 0x17, - CMD_QUERY_CQ = 0x18, - CMD_RESIZE_CQ = 0x2c, - - /* SRQ commands */ - CMD_SW2HW_SRQ = 0x35, - CMD_HW2SW_SRQ = 0x36, - CMD_QUERY_SRQ = 0x37, - CMD_ARM_SRQ = 0x40, - - /* QP/EE commands */ - CMD_RST2INIT_QPEE = 0x19, - CMD_INIT2RTR_QPEE = 0x1a, - CMD_RTR2RTS_QPEE = 0x1b, - CMD_RTS2RTS_QPEE = 0x1c, - CMD_SQERR2RTS_QPEE = 0x1d, - CMD_2ERR_QPEE = 0x1e, - CMD_RTS2SQD_QPEE = 0x1f, - CMD_SQD2SQD_QPEE = 0x38, - CMD_SQD2RTS_QPEE = 0x20, - CMD_ERR2RST_QPEE = 0x21, - CMD_QUERY_QPEE = 0x22, - CMD_INIT2INIT_QPEE = 0x2d, - CMD_SUSPEND_QPEE = 0x32, - CMD_UNSUSPEND_QPEE = 0x33, - /* special QPs and management commands */ - CMD_CONF_SPECIAL_QP = 0x23, - CMD_MAD_IFC = 0x24, - - /* multicast commands */ - CMD_READ_MGM = 0x25, - CMD_WRITE_MGM = 0x26, - CMD_MGID_HASH = 0x27, - - /* miscellaneous commands */ - CMD_DIAG_RPRT = 0x30, - CMD_NOP = 0x31, - - /* debug commands */ - CMD_QUERY_DEBUG_MSG = 0x2a, - CMD_SET_DEBUG_MSG = 0x2b, -}; - -/* - * According to Mellanox code, FW may be starved and never complete - * commands. So we can't use strict timeouts described in PRM -- we - * just arbitrarily select 60 seconds for now. - */ -#if 0 -/* - * Round up and add 1 to make sure we get the full wait time (since we - * will be starting in the middle of a jiffy) - */ -enum { - CMD_TIME_CLASS_A = (HZ + 999) / 1000 + 1, - CMD_TIME_CLASS_B = (HZ + 99) / 100 + 1, - CMD_TIME_CLASS_C = (HZ + 9) / 10 + 1, - CMD_TIME_CLASS_D = 60 * HZ -}; -#else -#define CMD_TIME_CLASS_A (60 * HZ) -#define CMD_TIME_CLASS_B (60 * HZ) -#define CMD_TIME_CLASS_C (60 * HZ) -#define CMD_TIME_CLASS_D (60 * HZ) -#endif - -#define GO_BIT_TIMEOUT (HZ * 10) - -struct mthca_cmd_context { - struct completion done; - int result; - int next; - u64 out_param; - u16 token; - u8 status; -}; - -static int fw_cmd_doorbell = 0; -module_param(fw_cmd_doorbell, int, 0644); -MODULE_PARM_DESC(fw_cmd_doorbell, "post FW commands through doorbell page if nonzero " - "(and supported by FW)"); - -static inline int go_bit(struct mthca_dev *dev) -{ - return readl(dev->hcr + HCR_STATUS_OFFSET) & - swab32(1 << HCR_GO_BIT); -} - -static void mthca_cmd_post_dbell(struct mthca_dev *dev, - u64 in_param, - u64 out_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - u16 token) -{ - void __iomem *ptr = dev->cmd.dbell_map; - u16 *offs = dev->cmd.dbell_offsets; - - __raw_writel((__force u32) cpu_to_be32(in_param >> 32), ptr + offs[0]); - wmb(); - __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), ptr + offs[1]); - wmb(); - __raw_writel((__force u32) cpu_to_be32(in_modifier), ptr + offs[2]); - wmb(); - __raw_writel((__force u32) cpu_to_be32(out_param >> 32), ptr + offs[3]); - wmb(); - __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), ptr + offs[4]); - wmb(); - __raw_writel((__force u32) cpu_to_be32(token << 16), ptr + offs[5]); - wmb(); - __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | - (1 << HCA_E_BIT) | - (op_modifier << HCR_OPMOD_SHIFT) | - op), ptr + offs[6]); - wmb(); - __raw_writel((__force u32) 0, ptr + offs[7]); - wmb(); -} - -static int mthca_cmd_post_hcr(struct mthca_dev *dev, - u64 in_param, - u64 out_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - u16 token, - int event) -{ - if (event) { - unsigned long end = jiffies + GO_BIT_TIMEOUT; - - while (go_bit(dev) && time_before(jiffies, end)) - sched_yield(); - } - - if (go_bit(dev)) - return -EAGAIN; - - /* - * We use writel (instead of something like memcpy_toio) - * because writes of less than 32 bits to the HCR don't work - * (and some architectures such as ia64 implement memcpy_toio - * in terms of writeb). - */ - __raw_writel((__force u32) cpu_to_be32(in_param >> 32), dev->hcr + 0 * 4); - __raw_writel((__force u32) cpu_to_be32(in_param & 0xfffffffful), dev->hcr + 1 * 4); - __raw_writel((__force u32) cpu_to_be32(in_modifier), dev->hcr + 2 * 4); - __raw_writel((__force u32) cpu_to_be32(out_param >> 32), dev->hcr + 3 * 4); - __raw_writel((__force u32) cpu_to_be32(out_param & 0xfffffffful), dev->hcr + 4 * 4); - __raw_writel((__force u32) cpu_to_be32(token << 16), dev->hcr + 5 * 4); - - /* __raw_writel may not order writes. */ - wmb(); - - __raw_writel((__force u32) cpu_to_be32((1 << HCR_GO_BIT) | - (event ? (1 << HCA_E_BIT) : 0) | - (op_modifier << HCR_OPMOD_SHIFT) | - op), dev->hcr + 6 * 4); - - return 0; -} - -static int mthca_cmd_post(struct mthca_dev *dev, - u64 in_param, - u64 out_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - u16 token, - int event) -{ - int err = 0; - - mutex_lock(&dev->cmd.hcr_mutex); - - if (event && dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS && fw_cmd_doorbell) - mthca_cmd_post_dbell(dev, in_param, out_param, in_modifier, - op_modifier, op, token); - else - err = mthca_cmd_post_hcr(dev, in_param, out_param, in_modifier, - op_modifier, op, token, event); - - /* - * Make sure that our HCR writes don't get mixed in with - * writes from another CPU starting a FW command. - */ - mmiowb(); - - mutex_unlock(&dev->cmd.hcr_mutex); - return err; -} - -static int mthca_cmd_poll(struct mthca_dev *dev, - u64 in_param, - u64 *out_param, - int out_is_imm, - u32 in_modifier, - u8 op_modifier, - u16 op, - unsigned long timeout, - u8 *status) -{ - int err = 0; - unsigned long end; - - down(&dev->cmd.poll_sem); - - err = mthca_cmd_post(dev, in_param, - out_param ? *out_param : 0, - in_modifier, op_modifier, - op, CMD_POLL_TOKEN, 0); - if (err) - goto out; - - end = timeout + jiffies; - while (go_bit(dev) && time_before(jiffies, end)) - sched_yield(); - - if (go_bit(dev)) { - err = -EBUSY; - goto out; - } - - if (out_is_imm) - *out_param = - (u64) be32_to_cpu((__force __be32) - __raw_readl(dev->hcr + HCR_OUT_PARAM_OFFSET)) << 32 | - (u64) be32_to_cpu((__force __be32) - __raw_readl(dev->hcr + HCR_OUT_PARAM_OFFSET + 4)); - - *status = be32_to_cpu((__force __be32) __raw_readl(dev->hcr + HCR_STATUS_OFFSET)) >> 24; - -out: - up(&dev->cmd.poll_sem); - return err; -} - -void mthca_cmd_event(struct mthca_dev *dev, - u16 token, - u8 status, - u64 out_param) -{ - struct mthca_cmd_context *context = - &dev->cmd.context[token & dev->cmd.token_mask]; - - /* previously timed out command completing at long last */ - if (token != context->token) - return; - - context->result = 0; - context->status = status; - context->out_param = out_param; - - complete(&context->done); -} - -static int mthca_cmd_wait(struct mthca_dev *dev, - u64 in_param, - u64 *out_param, - int out_is_imm, - u32 in_modifier, - u8 op_modifier, - u16 op, - unsigned long timeout, - u8 *status) -{ - int err = 0; - struct mthca_cmd_context *context; - - down(&dev->cmd.event_sem); - - spin_lock(&dev->cmd.context_lock); - BUG_ON(dev->cmd.free_head < 0); - context = &dev->cmd.context[dev->cmd.free_head]; - context->token += dev->cmd.token_mask + 1; - dev->cmd.free_head = context->next; - spin_unlock(&dev->cmd.context_lock); - - init_completion(&context->done); - - err = mthca_cmd_post(dev, in_param, - out_param ? *out_param : 0, - in_modifier, op_modifier, - op, context->token, 1); - if (err) - goto out; - - if (!wait_for_completion_timeout(&context->done, timeout)) { - err = -EBUSY; - goto out; - } - - err = context->result; - if (err) - goto out; - - *status = context->status; - if (*status) - mthca_dbg(dev, "Command %02x completed with status %02x\n", - op, *status); - - if (out_is_imm) - *out_param = context->out_param; - -out: - spin_lock(&dev->cmd.context_lock); - context->next = dev->cmd.free_head; - dev->cmd.free_head = context - dev->cmd.context; - spin_unlock(&dev->cmd.context_lock); - - up(&dev->cmd.event_sem); - return err; -} - -/* Invoke a command with an output mailbox */ -static int mthca_cmd_box(struct mthca_dev *dev, - u64 in_param, - u64 out_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - unsigned long timeout, - u8 *status) -{ - if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) - return mthca_cmd_wait(dev, in_param, &out_param, 0, - in_modifier, op_modifier, op, - timeout, status); - else - return mthca_cmd_poll(dev, in_param, &out_param, 0, - in_modifier, op_modifier, op, - timeout, status); -} - -/* Invoke a command with no output parameter */ -static int mthca_cmd(struct mthca_dev *dev, - u64 in_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - unsigned long timeout, - u8 *status) -{ - return mthca_cmd_box(dev, in_param, 0, in_modifier, - op_modifier, op, timeout, status); -} - -/* - * Invoke a command with an immediate output parameter (and copy the - * output into the caller's out_param pointer after the command - * executes). - */ -static int mthca_cmd_imm(struct mthca_dev *dev, - u64 in_param, - u64 *out_param, - u32 in_modifier, - u8 op_modifier, - u16 op, - unsigned long timeout, - u8 *status) -{ - if (dev->cmd.flags & MTHCA_CMD_USE_EVENTS) - return mthca_cmd_wait(dev, in_param, out_param, 1, - in_modifier, op_modifier, op, - timeout, status); - else - return mthca_cmd_poll(dev, in_param, out_param, 1, - in_modifier, op_modifier, op, - timeout, status); -} - -int mthca_cmd_init(struct mthca_dev *dev) -{ - mutex_init(&dev->cmd.hcr_mutex); - sema_init(&dev->cmd.poll_sem, 1); - dev->cmd.flags = 0; - - dev->hcr = ioremap(pci_resource_start(dev->pdev, 0) + MTHCA_HCR_BASE, - MTHCA_HCR_SIZE); - if (!dev->hcr) { - mthca_err(dev, "Couldn't map command register."); - return -ENOMEM; - } - - dev->cmd.pool = pci_pool_create("mthca_cmd", dev->pdev, - MTHCA_MAILBOX_SIZE, - MTHCA_MAILBOX_SIZE, 0); - if (!dev->cmd.pool) { - iounmap(dev->hcr); - return -ENOMEM; - } - - return 0; -} - -void mthca_cmd_cleanup(struct mthca_dev *dev) -{ - pci_pool_destroy(dev->cmd.pool); - iounmap(dev->hcr); - if (dev->cmd.flags & MTHCA_CMD_POST_DOORBELLS) - iounmap(dev->cmd.dbell_map); -} - -/* - * Switch to using events to issue FW commands (should be called after - * event queue to command events has been initialized). - */ -int mthca_cmd_use_events(struct mthca_dev *dev) -{ - int i; - - dev->cmd.context = kmalloc(dev->cmd.max_cmds * - sizeof (struct mthca_cmd_context), - GFP_KERNEL); - if (!dev->cmd.context) - return -ENOMEM; - - for (i = 0; i < dev->cmd.max_cmds; ++i) { - dev->cmd.context[i].token = i; - dev->cmd.context[i].next = i + 1; - } - - dev->cmd.context[dev->cmd.max_cmds - 1].next = -1; - dev->cmd.free_head = 0; - - sema_init(&dev->cmd.event_sem, dev->cmd.max_cmds); - spin_lock_init(&dev->cmd.context_lock); - - for (dev->cmd.token_mask = 1; - dev->cmd.token_mask < dev->cmd.max_cmds; - dev->cmd.token_mask <<= 1) - ; /* nothing */ - --dev->cmd.token_mask; - - dev->cmd.flags |= MTHCA_CMD_USE_EVENTS; - - down(&dev->cmd.poll_sem); - - return 0; -} - -/* - * Switch back to polling (used when shutting down the device) - */ -void mthca_cmd_use_polling(struct mthca_dev *dev) -{ - int i; - - dev->cmd.flags &= ~MTHCA_CMD_USE_EVENTS; - - for (i = 0; i < dev->cmd.max_cmds; ++i) - down(&dev->cmd.event_sem); - - kfree(dev->cmd.context); - - up(&dev->cmd.poll_sem); -} - -struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev, - gfp_t gfp_mask) -{ - struct mthca_mailbox *mailbox; - - mailbox = kmalloc(sizeof *mailbox, gfp_mask); - if (!mailbox) - return ERR_PTR(-ENOMEM); - - mailbox->buf = pci_pool_alloc(dev->cmd.pool, gfp_mask, &mailbox->dma); - if (!mailbox->buf) { - kfree(mailbox); - return ERR_PTR(-ENOMEM); - } - - return mailbox; -} - -void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox) -{ - if (!mailbox) - return; - - pci_pool_free(dev->cmd.pool, mailbox->buf, mailbox->dma); - kfree(mailbox); -} - -int mthca_SYS_EN(struct mthca_dev *dev, u8 *status) -{ - u64 out; - int ret; - - ret = mthca_cmd_imm(dev, 0, &out, 0, 0, CMD_SYS_EN, CMD_TIME_CLASS_D, status); - - if (*status == MTHCA_CMD_STAT_DDR_MEM_ERR) - mthca_warn(dev, "SYS_EN DDR error: syn=%x, sock=%d, " - "sladdr=%d, SPD source=%s\n", - (int) (out >> 6) & 0xf, (int) (out >> 4) & 3, - (int) (out >> 1) & 7, (int) out & 1 ? "NVMEM" : "DIMM"); - - return ret; -} - -int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status); -} - -static int mthca_map_cmd(struct mthca_dev *dev, u16 op, struct mthca_icm *icm, - u64 virt, u8 *status) -{ - struct mthca_mailbox *mailbox; - struct mthca_icm_iter iter; - __be64 *pages; - int lg; - int nent = 0; - int i; - int err = 0; - int ts = 0, tc = 0; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - memset(mailbox->buf, 0, MTHCA_MAILBOX_SIZE); - pages = mailbox->buf; - - for (mthca_icm_first(icm, &iter); - !mthca_icm_last(&iter); - mthca_icm_next(&iter)) { - /* - * We have to pass pages that are aligned to their - * size, so find the least significant 1 in the - * address or size and use that as our log2 size. - */ - lg = ffs(mthca_icm_addr(&iter) | mthca_icm_size(&iter)) - 1; - if (lg < MTHCA_ICM_PAGE_SHIFT) { - mthca_warn(dev, "Got FW area not aligned to %d (%llx/%lx).\n", - MTHCA_ICM_PAGE_SIZE, - (unsigned long long) mthca_icm_addr(&iter), - mthca_icm_size(&iter)); - err = -EINVAL; - goto out; - } - for (i = 0; i < mthca_icm_size(&iter) >> lg; ++i) { - if (virt != -1) { - pages[nent * 2] = cpu_to_be64(virt); - virt += 1 << lg; - } - - pages[nent * 2 + 1] = - cpu_to_be64((mthca_icm_addr(&iter) + (i << lg)) | - (lg - MTHCA_ICM_PAGE_SHIFT)); - ts += 1 << (lg - 10); - ++tc; - - if (++nent == MTHCA_MAILBOX_SIZE / 16) { - err = mthca_cmd(dev, mailbox->dma, nent, 0, op, - CMD_TIME_CLASS_B, status); - if (err || *status) - goto out; - nent = 0; - } - } - } - - if (nent) - err = mthca_cmd(dev, mailbox->dma, nent, 0, op, - CMD_TIME_CLASS_B, status); - - switch (op) { - case CMD_MAP_FA: - mthca_dbg(dev, "Mapped %d chunks/%d KB for FW.\n", tc, ts); - break; - case CMD_MAP_ICM_AUX: - mthca_dbg(dev, "Mapped %d chunks/%d KB for ICM aux.\n", tc, ts); - break; - case CMD_MAP_ICM: - mthca_dbg(dev, "Mapped %d chunks/%d KB at %llx for ICM.\n", - tc, ts, (unsigned long long) virt - (ts << 10)); - break; - } - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) -{ - return mthca_map_cmd(dev, CMD_MAP_FA, icm, -1, status); -} - -int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_FA, CMD_TIME_CLASS_B, status); -} - -int mthca_RUN_FW(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_RUN_FW, CMD_TIME_CLASS_A, status); -} - -static void mthca_setup_cmd_doorbells(struct mthca_dev *dev, u64 base) -{ - unsigned long addr; - u16 max_off = 0; - int i; - - for (i = 0; i < 8; ++i) - max_off = max(max_off, dev->cmd.dbell_offsets[i]); - - if ((base & PAGE_MASK) != ((base + max_off) & PAGE_MASK)) { - mthca_warn(dev, "Firmware doorbell region at 0x%016llx, " - "length 0x%x crosses a page boundary\n", - (unsigned long long) base, max_off); - return; - } - - addr = pci_resource_start(dev->pdev, 2) + - ((pci_resource_len(dev->pdev, 2) - 1) & base); - dev->cmd.dbell_map = ioremap(addr, max_off + sizeof(u32)); - if (!dev->cmd.dbell_map) - return; - - dev->cmd.flags |= MTHCA_CMD_POST_DOORBELLS; - mthca_dbg(dev, "Mapped doorbell page for posting FW commands\n"); -} - -int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status) -{ - struct mthca_mailbox *mailbox; - u32 *outbox; - u64 base; - u32 tmp; - int err = 0; - u8 lg; - int i; - -#define QUERY_FW_OUT_SIZE 0x100 -#define QUERY_FW_VER_OFFSET 0x00 -#define QUERY_FW_MAX_CMD_OFFSET 0x0f -#define QUERY_FW_ERR_START_OFFSET 0x30 -#define QUERY_FW_ERR_SIZE_OFFSET 0x38 - -#define QUERY_FW_CMD_DB_EN_OFFSET 0x10 -#define QUERY_FW_CMD_DB_OFFSET 0x50 -#define QUERY_FW_CMD_DB_BASE 0x60 - -#define QUERY_FW_START_OFFSET 0x20 -#define QUERY_FW_END_OFFSET 0x28 - -#define QUERY_FW_SIZE_OFFSET 0x00 -#define QUERY_FW_CLR_INT_BASE_OFFSET 0x20 -#define QUERY_FW_EQ_ARM_BASE_OFFSET 0x40 -#define QUERY_FW_EQ_SET_CI_BASE_OFFSET 0x48 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_FW, - CMD_TIME_CLASS_A, status); - - if (err) - goto out; - - MTHCA_GET(dev->fw_ver, outbox, QUERY_FW_VER_OFFSET); - /* - * FW subminor version is at more significant bits than minor - * version, so swap here. - */ - dev->fw_ver = (dev->fw_ver & 0xffff00000000ull) | - ((dev->fw_ver & 0xffff0000ull) >> 16) | - ((dev->fw_ver & 0x0000ffffull) << 16); - - MTHCA_GET(lg, outbox, QUERY_FW_MAX_CMD_OFFSET); - dev->cmd.max_cmds = 1 << lg; - - mthca_dbg(dev, "FW version %012llx, max commands %d\n", - (unsigned long long) dev->fw_ver, dev->cmd.max_cmds); - - MTHCA_GET(dev->catas_err.addr, outbox, QUERY_FW_ERR_START_OFFSET); - MTHCA_GET(dev->catas_err.size, outbox, QUERY_FW_ERR_SIZE_OFFSET); - - mthca_dbg(dev, "Catastrophic error buffer at 0x%llx, size 0x%x\n", - (unsigned long long) dev->catas_err.addr, dev->catas_err.size); - - MTHCA_GET(tmp, outbox, QUERY_FW_CMD_DB_EN_OFFSET); - if (tmp & 0x1) { - mthca_dbg(dev, "FW supports commands through doorbells\n"); - - MTHCA_GET(base, outbox, QUERY_FW_CMD_DB_BASE); - for (i = 0; i < MTHCA_CMD_NUM_DBELL_DWORDS; ++i) - MTHCA_GET(dev->cmd.dbell_offsets[i], outbox, - QUERY_FW_CMD_DB_OFFSET + (i << 1)); - - mthca_setup_cmd_doorbells(dev, base); - } - - if (mthca_is_memfree(dev)) { - MTHCA_GET(dev->fw.arbel.fw_pages, outbox, QUERY_FW_SIZE_OFFSET); - MTHCA_GET(dev->fw.arbel.clr_int_base, outbox, QUERY_FW_CLR_INT_BASE_OFFSET); - MTHCA_GET(dev->fw.arbel.eq_arm_base, outbox, QUERY_FW_EQ_ARM_BASE_OFFSET); - MTHCA_GET(dev->fw.arbel.eq_set_ci_base, outbox, QUERY_FW_EQ_SET_CI_BASE_OFFSET); - mthca_dbg(dev, "FW size %d KB\n", dev->fw.arbel.fw_pages << 2); - - /* - * Round up number of system pages needed in case - * MTHCA_ICM_PAGE_SIZE < PAGE_SIZE. - */ - dev->fw.arbel.fw_pages = - ALIGN(dev->fw.arbel.fw_pages, PAGE_SIZE / MTHCA_ICM_PAGE_SIZE) >> - (PAGE_SHIFT - MTHCA_ICM_PAGE_SHIFT); - - mthca_dbg(dev, "Clear int @ %llx, EQ arm @ %llx, EQ set CI @ %llx\n", - (unsigned long long) dev->fw.arbel.clr_int_base, - (unsigned long long) dev->fw.arbel.eq_arm_base, - (unsigned long long) dev->fw.arbel.eq_set_ci_base); - } else { - MTHCA_GET(dev->fw.tavor.fw_start, outbox, QUERY_FW_START_OFFSET); - MTHCA_GET(dev->fw.tavor.fw_end, outbox, QUERY_FW_END_OFFSET); - - mthca_dbg(dev, "FW size %d KB (start %llx, end %llx)\n", - (int) ((dev->fw.tavor.fw_end - dev->fw.tavor.fw_start) >> 10), - (unsigned long long) dev->fw.tavor.fw_start, - (unsigned long long) dev->fw.tavor.fw_end); - } - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status) -{ - struct mthca_mailbox *mailbox; - u8 info; - u32 *outbox; - int err = 0; - -#define ENABLE_LAM_OUT_SIZE 0x100 -#define ENABLE_LAM_START_OFFSET 0x00 -#define ENABLE_LAM_END_OFFSET 0x08 -#define ENABLE_LAM_INFO_OFFSET 0x13 - -#define ENABLE_LAM_INFO_HIDDEN_FLAG (1 << 4) -#define ENABLE_LAM_INFO_ECC_MASK 0x3 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_ENABLE_LAM, - CMD_TIME_CLASS_C, status); - - if (err) - goto out; - - if (*status == MTHCA_CMD_STAT_LAM_NOT_PRE) - goto out; - - MTHCA_GET(dev->ddr_start, outbox, ENABLE_LAM_START_OFFSET); - MTHCA_GET(dev->ddr_end, outbox, ENABLE_LAM_END_OFFSET); - MTHCA_GET(info, outbox, ENABLE_LAM_INFO_OFFSET); - - if (!!(info & ENABLE_LAM_INFO_HIDDEN_FLAG) != - !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { - mthca_info(dev, "FW reports that HCA-attached memory " - "is %s hidden; does not match PCI config\n", - (info & ENABLE_LAM_INFO_HIDDEN_FLAG) ? - "" : "not"); - } - if (info & ENABLE_LAM_INFO_HIDDEN_FLAG) - mthca_dbg(dev, "HCA-attached memory is hidden.\n"); - - mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n", - (int) ((dev->ddr_end - dev->ddr_start) >> 10), - (unsigned long long) dev->ddr_start, - (unsigned long long) dev->ddr_end); - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_SYS_DIS, CMD_TIME_CLASS_C, status); -} - -int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status) -{ - struct mthca_mailbox *mailbox; - u8 info; - u32 *outbox; - int err = 0; - -#define QUERY_DDR_OUT_SIZE 0x100 -#define QUERY_DDR_START_OFFSET 0x00 -#define QUERY_DDR_END_OFFSET 0x08 -#define QUERY_DDR_INFO_OFFSET 0x13 - -#define QUERY_DDR_INFO_HIDDEN_FLAG (1 << 4) -#define QUERY_DDR_INFO_ECC_MASK 0x3 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DDR, - CMD_TIME_CLASS_A, status); - - if (err) - goto out; - - MTHCA_GET(dev->ddr_start, outbox, QUERY_DDR_START_OFFSET); - MTHCA_GET(dev->ddr_end, outbox, QUERY_DDR_END_OFFSET); - MTHCA_GET(info, outbox, QUERY_DDR_INFO_OFFSET); - - if (!!(info & QUERY_DDR_INFO_HIDDEN_FLAG) != - !!(dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) { - mthca_info(dev, "FW reports that HCA-attached memory " - "is %s hidden; does not match PCI config\n", - (info & QUERY_DDR_INFO_HIDDEN_FLAG) ? - "" : "not"); - } - if (info & QUERY_DDR_INFO_HIDDEN_FLAG) - mthca_dbg(dev, "HCA-attached memory is hidden.\n"); - - mthca_dbg(dev, "HCA memory size %d KB (start %llx, end %llx)\n", - (int) ((dev->ddr_end - dev->ddr_start) >> 10), - (unsigned long long) dev->ddr_start, - (unsigned long long) dev->ddr_end); - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, - struct mthca_dev_lim *dev_lim, u8 *status) -{ - struct mthca_mailbox *mailbox; - u32 *outbox; - u8 field; - u16 size; - u16 stat_rate; - int err; - -#define QUERY_DEV_LIM_OUT_SIZE 0x100 -#define QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET 0x10 -#define QUERY_DEV_LIM_MAX_QP_SZ_OFFSET 0x11 -#define QUERY_DEV_LIM_RSVD_QP_OFFSET 0x12 -#define QUERY_DEV_LIM_MAX_QP_OFFSET 0x13 -#define QUERY_DEV_LIM_RSVD_SRQ_OFFSET 0x14 -#define QUERY_DEV_LIM_MAX_SRQ_OFFSET 0x15 -#define QUERY_DEV_LIM_RSVD_EEC_OFFSET 0x16 -#define QUERY_DEV_LIM_MAX_EEC_OFFSET 0x17 -#define QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET 0x19 -#define QUERY_DEV_LIM_RSVD_CQ_OFFSET 0x1a -#define QUERY_DEV_LIM_MAX_CQ_OFFSET 0x1b -#define QUERY_DEV_LIM_MAX_MPT_OFFSET 0x1d -#define QUERY_DEV_LIM_RSVD_EQ_OFFSET 0x1e -#define QUERY_DEV_LIM_MAX_EQ_OFFSET 0x1f -#define QUERY_DEV_LIM_RSVD_MTT_OFFSET 0x20 -#define QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET 0x21 -#define QUERY_DEV_LIM_RSVD_MRW_OFFSET 0x22 -#define QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET 0x23 -#define QUERY_DEV_LIM_MAX_AV_OFFSET 0x27 -#define QUERY_DEV_LIM_MAX_REQ_QP_OFFSET 0x29 -#define QUERY_DEV_LIM_MAX_RES_QP_OFFSET 0x2b -#define QUERY_DEV_LIM_MAX_RDMA_OFFSET 0x2f -#define QUERY_DEV_LIM_RSZ_SRQ_OFFSET 0x33 -#define QUERY_DEV_LIM_ACK_DELAY_OFFSET 0x35 -#define QUERY_DEV_LIM_MTU_WIDTH_OFFSET 0x36 -#define QUERY_DEV_LIM_VL_PORT_OFFSET 0x37 -#define QUERY_DEV_LIM_MAX_GID_OFFSET 0x3b -#define QUERY_DEV_LIM_RATE_SUPPORT_OFFSET 0x3c -#define QUERY_DEV_LIM_MAX_PKEY_OFFSET 0x3f -#define QUERY_DEV_LIM_FLAGS_OFFSET 0x44 -#define QUERY_DEV_LIM_RSVD_UAR_OFFSET 0x48 -#define QUERY_DEV_LIM_UAR_SZ_OFFSET 0x49 -#define QUERY_DEV_LIM_PAGE_SZ_OFFSET 0x4b -#define QUERY_DEV_LIM_MAX_SG_OFFSET 0x51 -#define QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET 0x52 -#define QUERY_DEV_LIM_MAX_SG_RQ_OFFSET 0x55 -#define QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET 0x56 -#define QUERY_DEV_LIM_MAX_QP_MCG_OFFSET 0x61 -#define QUERY_DEV_LIM_RSVD_MCG_OFFSET 0x62 -#define QUERY_DEV_LIM_MAX_MCG_OFFSET 0x63 -#define QUERY_DEV_LIM_RSVD_PD_OFFSET 0x64 -#define QUERY_DEV_LIM_MAX_PD_OFFSET 0x65 -#define QUERY_DEV_LIM_RSVD_RDD_OFFSET 0x66 -#define QUERY_DEV_LIM_MAX_RDD_OFFSET 0x67 -#define QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET 0x80 -#define QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET 0x82 -#define QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET 0x84 -#define QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET 0x86 -#define QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET 0x88 -#define QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET 0x8a -#define QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET 0x8c -#define QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET 0x8e -#define QUERY_DEV_LIM_MTT_ENTRY_SZ_OFFSET 0x90 -#define QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET 0x92 -#define QUERY_DEV_LIM_PBL_SZ_OFFSET 0x96 -#define QUERY_DEV_LIM_BMME_FLAGS_OFFSET 0x97 -#define QUERY_DEV_LIM_RSVD_LKEY_OFFSET 0x98 -#define QUERY_DEV_LIM_LAMR_OFFSET 0x9f -#define QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET 0xa0 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_DEV_LIM, - CMD_TIME_CLASS_A, status); - - if (err) - goto out; - - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_QP_OFFSET); - dev_lim->reserved_qps = 1 << (field & 0xf); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_OFFSET); - dev_lim->max_qps = 1 << (field & 0x1f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_SRQ_OFFSET); - dev_lim->reserved_srqs = 1 << (field >> 4); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_OFFSET); - dev_lim->max_srqs = 1 << (field & 0x1f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EEC_OFFSET); - dev_lim->reserved_eecs = 1 << (field & 0xf); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EEC_OFFSET); - dev_lim->max_eecs = 1 << (field & 0x1f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_SZ_OFFSET); - dev_lim->max_cq_sz = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_CQ_OFFSET); - dev_lim->reserved_cqs = 1 << (field & 0xf); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_CQ_OFFSET); - dev_lim->max_cqs = 1 << (field & 0x1f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MPT_OFFSET); - dev_lim->max_mpts = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_EQ_OFFSET); - dev_lim->reserved_eqs = 1 << (field & 0xf); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_EQ_OFFSET); - dev_lim->max_eqs = 1 << (field & 0x7); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MTT_OFFSET); - if (mthca_is_memfree(dev)) - dev_lim->reserved_mtts = ALIGN((1 << (field >> 4)) * sizeof(u64), - dev->limits.mtt_seg_size) / dev->limits.mtt_seg_size; - else - dev_lim->reserved_mtts = 1 << (field >> 4); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MRW_SZ_OFFSET); - dev_lim->max_mrw_sz = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MRW_OFFSET); - dev_lim->reserved_mrws = 1 << (field & 0xf); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MTT_SEG_OFFSET); - dev_lim->max_mtt_seg = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_REQ_QP_OFFSET); - dev_lim->max_requester_per_qp = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RES_QP_OFFSET); - dev_lim->max_responder_per_qp = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDMA_OFFSET); - dev_lim->max_rdma_global = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_ACK_DELAY_OFFSET); - dev_lim->local_ca_ack_delay = field & 0x1f; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MTU_WIDTH_OFFSET); - dev_lim->max_mtu = field >> 4; - dev_lim->max_port_width = field & 0xf; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_VL_PORT_OFFSET); - dev_lim->max_vl = field >> 4; - dev_lim->num_ports = field & 0xf; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_GID_OFFSET); - dev_lim->max_gids = 1 << (field & 0xf); - MTHCA_GET(stat_rate, outbox, QUERY_DEV_LIM_RATE_SUPPORT_OFFSET); - dev_lim->stat_rate_support = stat_rate; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PKEY_OFFSET); - dev_lim->max_pkeys = 1 << (field & 0xf); - MTHCA_GET(dev_lim->flags, outbox, QUERY_DEV_LIM_FLAGS_OFFSET); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_UAR_OFFSET); - dev_lim->reserved_uars = field >> 4; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_UAR_SZ_OFFSET); - dev_lim->uar_size = 1 << ((field & 0x3f) + 20); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_PAGE_SZ_OFFSET); - dev_lim->min_page_sz = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_OFFSET); - dev_lim->max_sg = field; - - MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_OFFSET); - dev_lim->max_desc_sz = size; - - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_MCG_OFFSET); - dev_lim->max_qp_per_mcg = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_MCG_OFFSET); - dev_lim->reserved_mgms = field & 0xf; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_MCG_OFFSET); - dev_lim->max_mcgs = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_PD_OFFSET); - dev_lim->reserved_pds = field >> 4; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_PD_OFFSET); - dev_lim->max_pds = 1 << (field & 0x3f); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSVD_RDD_OFFSET); - dev_lim->reserved_rdds = field >> 4; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_RDD_OFFSET); - dev_lim->max_rdds = 1 << (field & 0x3f); - - MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEC_ENTRY_SZ_OFFSET); - dev_lim->eec_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_QPC_ENTRY_SZ_OFFSET); - dev_lim->qpc_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_EEEC_ENTRY_SZ_OFFSET); - dev_lim->eeec_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQPC_ENTRY_SZ_OFFSET); - dev_lim->eqpc_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_EQC_ENTRY_SZ_OFFSET); - dev_lim->eqc_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_CQC_ENTRY_SZ_OFFSET); - dev_lim->cqc_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_SRQ_ENTRY_SZ_OFFSET); - dev_lim->srq_entry_sz = size; - MTHCA_GET(size, outbox, QUERY_DEV_LIM_UAR_ENTRY_SZ_OFFSET); - dev_lim->uar_scratch_entry_sz = size; - - if (mthca_is_memfree(dev)) { - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET); - dev_lim->max_srq_sz = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET); - dev_lim->max_qp_sz = 1 << field; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_RSZ_SRQ_OFFSET); - dev_lim->hca.arbel.resize_srq = field & 1; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SG_RQ_OFFSET); - dev_lim->max_sg = min_t(int, field, dev_lim->max_sg); - MTHCA_GET(size, outbox, QUERY_DEV_LIM_MAX_DESC_SZ_RQ_OFFSET); - dev_lim->max_desc_sz = min_t(int, size, dev_lim->max_desc_sz); - MTHCA_GET(size, outbox, QUERY_DEV_LIM_MPT_ENTRY_SZ_OFFSET); - dev_lim->mpt_entry_sz = size; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_PBL_SZ_OFFSET); - dev_lim->hca.arbel.max_pbl_sz = 1 << (field & 0x3f); - MTHCA_GET(dev_lim->hca.arbel.bmme_flags, outbox, - QUERY_DEV_LIM_BMME_FLAGS_OFFSET); - MTHCA_GET(dev_lim->hca.arbel.reserved_lkey, outbox, - QUERY_DEV_LIM_RSVD_LKEY_OFFSET); - MTHCA_GET(field, outbox, QUERY_DEV_LIM_LAMR_OFFSET); - dev_lim->hca.arbel.lam_required = field & 1; - MTHCA_GET(dev_lim->hca.arbel.max_icm_sz, outbox, - QUERY_DEV_LIM_MAX_ICM_SZ_OFFSET); - - if (dev_lim->hca.arbel.bmme_flags & 1) - mthca_dbg(dev, "Base MM extensions: yes " - "(flags %d, max PBL %d, rsvd L_Key %08x)\n", - dev_lim->hca.arbel.bmme_flags, - dev_lim->hca.arbel.max_pbl_sz, - dev_lim->hca.arbel.reserved_lkey); - else - mthca_dbg(dev, "Base MM extensions: no\n"); - - mthca_dbg(dev, "Max ICM size %lld MB\n", - (unsigned long long) dev_lim->hca.arbel.max_icm_sz >> 20); - } else { - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_SRQ_SZ_OFFSET); - dev_lim->max_srq_sz = (1 << field) - 1; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_QP_SZ_OFFSET); - dev_lim->max_qp_sz = (1 << field) - 1; - MTHCA_GET(field, outbox, QUERY_DEV_LIM_MAX_AV_OFFSET); - dev_lim->hca.tavor.max_avs = 1 << (field & 0x3f); - dev_lim->mpt_entry_sz = MTHCA_MPT_ENTRY_SIZE; - } - - mthca_dbg(dev, "Max QPs: %d, reserved QPs: %d, entry size: %d\n", - dev_lim->max_qps, dev_lim->reserved_qps, dev_lim->qpc_entry_sz); - mthca_dbg(dev, "Max SRQs: %d, reserved SRQs: %d, entry size: %d\n", - dev_lim->max_srqs, dev_lim->reserved_srqs, dev_lim->srq_entry_sz); - mthca_dbg(dev, "Max CQs: %d, reserved CQs: %d, entry size: %d\n", - dev_lim->max_cqs, dev_lim->reserved_cqs, dev_lim->cqc_entry_sz); - mthca_dbg(dev, "Max EQs: %d, reserved EQs: %d, entry size: %d\n", - dev_lim->max_eqs, dev_lim->reserved_eqs, dev_lim->eqc_entry_sz); - mthca_dbg(dev, "reserved MPTs: %d, reserved MTTs: %d\n", - dev_lim->reserved_mrws, dev_lim->reserved_mtts); - mthca_dbg(dev, "Max PDs: %d, reserved PDs: %d, reserved UARs: %d\n", - dev_lim->max_pds, dev_lim->reserved_pds, dev_lim->reserved_uars); - mthca_dbg(dev, "Max QP/MCG: %d, reserved MGMs: %d\n", - dev_lim->max_pds, dev_lim->reserved_mgms); - mthca_dbg(dev, "Max CQEs: %d, max WQEs: %d, max SRQ WQEs: %d\n", - dev_lim->max_cq_sz, dev_lim->max_qp_sz, dev_lim->max_srq_sz); - - mthca_dbg(dev, "Flags: %08x\n", dev_lim->flags); - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -static void get_board_id(void *vsd, char *board_id) -{ - int i; - -#define VSD_OFFSET_SIG1 0x00 -#define VSD_OFFSET_SIG2 0xde -#define VSD_OFFSET_MLX_BOARD_ID 0xd0 -#define VSD_OFFSET_TS_BOARD_ID 0x20 - -#define VSD_SIGNATURE_TOPSPIN 0x5ad - - memset(board_id, 0, MTHCA_BOARD_ID_LEN); - - if (be16_to_cpup(vsd + VSD_OFFSET_SIG1) == VSD_SIGNATURE_TOPSPIN && - be16_to_cpup(vsd + VSD_OFFSET_SIG2) == VSD_SIGNATURE_TOPSPIN) { - strlcpy(board_id, vsd + VSD_OFFSET_TS_BOARD_ID, MTHCA_BOARD_ID_LEN); - } else { - /* - * The board ID is a string but the firmware byte - * swaps each 4-byte word before passing it back to - * us. Therefore we need to swab it before printing. - */ - for (i = 0; i < 4; ++i) - ((u32 *) board_id)[i] = - swab32(*(u32 *) (vsd + VSD_OFFSET_MLX_BOARD_ID + i * 4)); - } -} - -int mthca_QUERY_ADAPTER(struct mthca_dev *dev, - struct mthca_adapter *adapter, u8 *status) -{ - struct mthca_mailbox *mailbox; - u32 *outbox; - int err; - -#define QUERY_ADAPTER_OUT_SIZE 0x100 -#define QUERY_ADAPTER_VENDOR_ID_OFFSET 0x00 -#define QUERY_ADAPTER_DEVICE_ID_OFFSET 0x04 -#define QUERY_ADAPTER_REVISION_ID_OFFSET 0x08 -#define QUERY_ADAPTER_INTA_PIN_OFFSET 0x10 -#define QUERY_ADAPTER_VSD_OFFSET 0x20 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - outbox = mailbox->buf; - - err = mthca_cmd_box(dev, 0, mailbox->dma, 0, 0, CMD_QUERY_ADAPTER, - CMD_TIME_CLASS_A, status); - - if (err) - goto out; - - if (!mthca_is_memfree(dev)) { - MTHCA_GET(adapter->vendor_id, outbox, - QUERY_ADAPTER_VENDOR_ID_OFFSET); - MTHCA_GET(adapter->device_id, outbox, - QUERY_ADAPTER_DEVICE_ID_OFFSET); - MTHCA_GET(adapter->revision_id, outbox, - QUERY_ADAPTER_REVISION_ID_OFFSET); - } - MTHCA_GET(adapter->inta_pin, outbox, QUERY_ADAPTER_INTA_PIN_OFFSET); - - get_board_id(outbox + QUERY_ADAPTER_VSD_OFFSET / 4, - adapter->board_id); - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_INIT_HCA(struct mthca_dev *dev, - struct mthca_init_hca_param *param, - u8 *status) -{ - struct mthca_mailbox *mailbox; - __be32 *inbox; - int err; - -#define INIT_HCA_IN_SIZE 0x200 -#define INIT_HCA_FLAGS1_OFFSET 0x00c -#define INIT_HCA_FLAGS2_OFFSET 0x014 -#define INIT_HCA_QPC_OFFSET 0x020 -#define INIT_HCA_QPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x10) -#define INIT_HCA_LOG_QP_OFFSET (INIT_HCA_QPC_OFFSET + 0x17) -#define INIT_HCA_EEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x20) -#define INIT_HCA_LOG_EEC_OFFSET (INIT_HCA_QPC_OFFSET + 0x27) -#define INIT_HCA_SRQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x28) -#define INIT_HCA_LOG_SRQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x2f) -#define INIT_HCA_CQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x30) -#define INIT_HCA_LOG_CQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x37) -#define INIT_HCA_EQPC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x40) -#define INIT_HCA_EEEC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x50) -#define INIT_HCA_EQC_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x60) -#define INIT_HCA_LOG_EQ_OFFSET (INIT_HCA_QPC_OFFSET + 0x67) -#define INIT_HCA_RDB_BASE_OFFSET (INIT_HCA_QPC_OFFSET + 0x70) -#define INIT_HCA_UDAV_OFFSET 0x0b0 -#define INIT_HCA_UDAV_LKEY_OFFSET (INIT_HCA_UDAV_OFFSET + 0x0) -#define INIT_HCA_UDAV_PD_OFFSET (INIT_HCA_UDAV_OFFSET + 0x4) -#define INIT_HCA_MCAST_OFFSET 0x0c0 -#define INIT_HCA_MC_BASE_OFFSET (INIT_HCA_MCAST_OFFSET + 0x00) -#define INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x12) -#define INIT_HCA_MC_HASH_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x16) -#define INIT_HCA_LOG_MC_TABLE_SZ_OFFSET (INIT_HCA_MCAST_OFFSET + 0x1b) -#define INIT_HCA_TPT_OFFSET 0x0f0 -#define INIT_HCA_MPT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x00) -#define INIT_HCA_MTT_SEG_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x09) -#define INIT_HCA_LOG_MPT_SZ_OFFSET (INIT_HCA_TPT_OFFSET + 0x0b) -#define INIT_HCA_MTT_BASE_OFFSET (INIT_HCA_TPT_OFFSET + 0x10) -#define INIT_HCA_UAR_OFFSET 0x120 -#define INIT_HCA_UAR_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x00) -#define INIT_HCA_UARC_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x09) -#define INIT_HCA_LOG_UAR_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0a) -#define INIT_HCA_UAR_PAGE_SZ_OFFSET (INIT_HCA_UAR_OFFSET + 0x0b) -#define INIT_HCA_UAR_SCATCH_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x10) -#define INIT_HCA_UAR_CTX_BASE_OFFSET (INIT_HCA_UAR_OFFSET + 0x18) - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, INIT_HCA_IN_SIZE); - - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - MTHCA_PUT(inbox, 0x1, INIT_HCA_FLAGS1_OFFSET); - -#if defined(__LITTLE_ENDIAN) - *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) &= ~cpu_to_be32(1 << 1); -#elif defined(__BIG_ENDIAN) - *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(1 << 1); -#else -#error Host endianness not defined -#endif - /* Check port for UD address vector: */ - *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(1); - - /* Enable IPoIB checksumming if we can: */ - if (dev->device_cap_flags & IB_DEVICE_UD_IP_CSUM) - *(inbox + INIT_HCA_FLAGS2_OFFSET / 4) |= cpu_to_be32(7 << 3); - - /* We leave wqe_quota, responder_exu, etc as 0 (default) */ - - /* QPC/EEC/CQC/EQC/RDB attributes */ - - MTHCA_PUT(inbox, param->qpc_base, INIT_HCA_QPC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_num_qps, INIT_HCA_LOG_QP_OFFSET); - MTHCA_PUT(inbox, param->eec_base, INIT_HCA_EEC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_num_eecs, INIT_HCA_LOG_EEC_OFFSET); - MTHCA_PUT(inbox, param->srqc_base, INIT_HCA_SRQC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_num_srqs, INIT_HCA_LOG_SRQ_OFFSET); - MTHCA_PUT(inbox, param->cqc_base, INIT_HCA_CQC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_num_cqs, INIT_HCA_LOG_CQ_OFFSET); - MTHCA_PUT(inbox, param->eqpc_base, INIT_HCA_EQPC_BASE_OFFSET); - MTHCA_PUT(inbox, param->eeec_base, INIT_HCA_EEEC_BASE_OFFSET); - MTHCA_PUT(inbox, param->eqc_base, INIT_HCA_EQC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_num_eqs, INIT_HCA_LOG_EQ_OFFSET); - MTHCA_PUT(inbox, param->rdb_base, INIT_HCA_RDB_BASE_OFFSET); - - /* UD AV attributes */ - - /* multicast attributes */ - - MTHCA_PUT(inbox, param->mc_base, INIT_HCA_MC_BASE_OFFSET); - MTHCA_PUT(inbox, param->log_mc_entry_sz, INIT_HCA_LOG_MC_ENTRY_SZ_OFFSET); - MTHCA_PUT(inbox, param->mc_hash_sz, INIT_HCA_MC_HASH_SZ_OFFSET); - MTHCA_PUT(inbox, param->log_mc_table_sz, INIT_HCA_LOG_MC_TABLE_SZ_OFFSET); - - /* TPT attributes */ - - MTHCA_PUT(inbox, param->mpt_base, INIT_HCA_MPT_BASE_OFFSET); - if (!mthca_is_memfree(dev)) - MTHCA_PUT(inbox, param->mtt_seg_sz, INIT_HCA_MTT_SEG_SZ_OFFSET); - MTHCA_PUT(inbox, param->log_mpt_sz, INIT_HCA_LOG_MPT_SZ_OFFSET); - MTHCA_PUT(inbox, param->mtt_base, INIT_HCA_MTT_BASE_OFFSET); - - /* UAR attributes */ - { - u8 uar_page_sz = PAGE_SHIFT - 12; - MTHCA_PUT(inbox, uar_page_sz, INIT_HCA_UAR_PAGE_SZ_OFFSET); - } - - MTHCA_PUT(inbox, param->uar_scratch_base, INIT_HCA_UAR_SCATCH_BASE_OFFSET); - - if (mthca_is_memfree(dev)) { - MTHCA_PUT(inbox, param->log_uarc_sz, INIT_HCA_UARC_SZ_OFFSET); - MTHCA_PUT(inbox, param->log_uar_sz, INIT_HCA_LOG_UAR_SZ_OFFSET); - MTHCA_PUT(inbox, param->uarc_base, INIT_HCA_UAR_CTX_BASE_OFFSET); - } - - err = mthca_cmd(dev, mailbox->dma, 0, 0, CMD_INIT_HCA, CMD_TIME_CLASS_D, status); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_INIT_IB(struct mthca_dev *dev, - struct mthca_init_ib_param *param, - int port, u8 *status) -{ - struct mthca_mailbox *mailbox; - u32 *inbox; - int err; - u32 flags; - -#define INIT_IB_IN_SIZE 56 -#define INIT_IB_FLAGS_OFFSET 0x00 -#define INIT_IB_FLAG_SIG (1 << 18) -#define INIT_IB_FLAG_NG (1 << 17) -#define INIT_IB_FLAG_G0 (1 << 16) -#define INIT_IB_VL_SHIFT 4 -#define INIT_IB_PORT_WIDTH_SHIFT 8 -#define INIT_IB_MTU_SHIFT 12 -#define INIT_IB_MAX_GID_OFFSET 0x06 -#define INIT_IB_MAX_PKEY_OFFSET 0x0a -#define INIT_IB_GUID0_OFFSET 0x10 -#define INIT_IB_NODE_GUID_OFFSET 0x18 -#define INIT_IB_SI_GUID_OFFSET 0x20 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, INIT_IB_IN_SIZE); - - flags = 0; - flags |= param->set_guid0 ? INIT_IB_FLAG_G0 : 0; - flags |= param->set_node_guid ? INIT_IB_FLAG_NG : 0; - flags |= param->set_si_guid ? INIT_IB_FLAG_SIG : 0; - flags |= param->vl_cap << INIT_IB_VL_SHIFT; - flags |= param->port_width << INIT_IB_PORT_WIDTH_SHIFT; - flags |= param->mtu_cap << INIT_IB_MTU_SHIFT; - MTHCA_PUT(inbox, flags, INIT_IB_FLAGS_OFFSET); - - MTHCA_PUT(inbox, param->gid_cap, INIT_IB_MAX_GID_OFFSET); - MTHCA_PUT(inbox, param->pkey_cap, INIT_IB_MAX_PKEY_OFFSET); - MTHCA_PUT(inbox, param->guid0, INIT_IB_GUID0_OFFSET); - MTHCA_PUT(inbox, param->node_guid, INIT_IB_NODE_GUID_OFFSET); - MTHCA_PUT(inbox, param->si_guid, INIT_IB_SI_GUID_OFFSET); - - err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_INIT_IB, - CMD_TIME_CLASS_A, status); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status) -{ - return mthca_cmd(dev, 0, port, 0, CMD_CLOSE_IB, CMD_TIME_CLASS_A, status); -} - -int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status) -{ - return mthca_cmd(dev, 0, 0, panic, CMD_CLOSE_HCA, CMD_TIME_CLASS_C, status); -} - -int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, - int port, u8 *status) -{ - struct mthca_mailbox *mailbox; - u32 *inbox; - int err; - u32 flags = 0; - -#define SET_IB_IN_SIZE 0x40 -#define SET_IB_FLAGS_OFFSET 0x00 -#define SET_IB_FLAG_SIG (1 << 18) -#define SET_IB_FLAG_RQK (1 << 0) -#define SET_IB_CAP_MASK_OFFSET 0x04 -#define SET_IB_SI_GUID_OFFSET 0x08 - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, SET_IB_IN_SIZE); - - flags |= param->set_si_guid ? SET_IB_FLAG_SIG : 0; - flags |= param->reset_qkey_viol ? SET_IB_FLAG_RQK : 0; - MTHCA_PUT(inbox, flags, SET_IB_FLAGS_OFFSET); - - MTHCA_PUT(inbox, param->cap_mask, SET_IB_CAP_MASK_OFFSET); - MTHCA_PUT(inbox, param->si_guid, SET_IB_SI_GUID_OFFSET); - - err = mthca_cmd(dev, mailbox->dma, port, 0, CMD_SET_IB, - CMD_TIME_CLASS_B, status); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status) -{ - return mthca_map_cmd(dev, CMD_MAP_ICM, icm, virt, status); -} - -int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status) -{ - struct mthca_mailbox *mailbox; - __be64 *inbox; - int err; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - inbox[0] = cpu_to_be64(virt); - inbox[1] = cpu_to_be64(dma_addr); - - err = mthca_cmd(dev, mailbox->dma, 1, 0, CMD_MAP_ICM, - CMD_TIME_CLASS_B, status); - - mthca_free_mailbox(dev, mailbox); - - if (!err) - mthca_dbg(dev, "Mapped page at %llx to %llx for ICM.\n", - (unsigned long long) dma_addr, (unsigned long long) virt); - - return err; -} - -int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status) -{ - mthca_dbg(dev, "Unmapping %d pages at %llx from ICM.\n", - page_count, (unsigned long long) virt); - - return mthca_cmd(dev, virt, page_count, 0, CMD_UNMAP_ICM, CMD_TIME_CLASS_B, status); -} - -int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status) -{ - return mthca_map_cmd(dev, CMD_MAP_ICM_AUX, icm, -1, status); -} - -int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_UNMAP_ICM_AUX, CMD_TIME_CLASS_B, status); -} - -int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, - u8 *status) -{ - int ret = mthca_cmd_imm(dev, icm_size, aux_pages, 0, 0, CMD_SET_ICM_SIZE, - CMD_TIME_CLASS_A, status); - - if (ret || status) - return ret; - - /* - * Round up number of system pages needed in case - * MTHCA_ICM_PAGE_SIZE < PAGE_SIZE. - */ - *aux_pages = ALIGN(*aux_pages, PAGE_SIZE / MTHCA_ICM_PAGE_SIZE) >> - (PAGE_SHIFT - MTHCA_ICM_PAGE_SHIFT); - - return 0; -} - -int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, mpt_index, 0, CMD_SW2HW_MPT, - CMD_TIME_CLASS_B, status); -} - -int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, - !mailbox, CMD_HW2SW_MPT, - CMD_TIME_CLASS_B, status); -} - -int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int num_mtt, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, num_mtt, 0, CMD_WRITE_MTT, - CMD_TIME_CLASS_B, status); -} - -int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0, 0, CMD_SYNC_TPT, CMD_TIME_CLASS_B, status); -} - -int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap, - int eq_num, u8 *status) -{ - mthca_dbg(dev, "%s mask %016llx for eqn %d\n", - unmap ? "Clearing" : "Setting", - (unsigned long long) event_mask, eq_num); - return mthca_cmd(dev, event_mask, (unmap << 31) | eq_num, - 0, CMD_MAP_EQ, CMD_TIME_CLASS_B, status); -} - -int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, eq_num, 0, CMD_SW2HW_EQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, eq_num, 0, - CMD_HW2SW_EQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, cq_num, 0, CMD_SW2HW_CQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, cq_num, 0, - CMD_HW2SW_CQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, - u8 *status) -{ - struct mthca_mailbox *mailbox; - __be32 *inbox; - int err; - -#define RESIZE_CQ_IN_SIZE 0x40 -#define RESIZE_CQ_LOG_SIZE_OFFSET 0x0c -#define RESIZE_CQ_LKEY_OFFSET 0x1c - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - inbox = mailbox->buf; - - memset(inbox, 0, RESIZE_CQ_IN_SIZE); - /* - * Leave start address fields zeroed out -- mthca assumes that - * MRs for CQs always start at virtual address 0. - */ - MTHCA_PUT(inbox, log_size, RESIZE_CQ_LOG_SIZE_OFFSET); - MTHCA_PUT(inbox, lkey, RESIZE_CQ_LKEY_OFFSET); - - err = mthca_cmd(dev, mailbox->dma, cq_num, 1, CMD_RESIZE_CQ, - CMD_TIME_CLASS_B, status); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, srq_num, 0, CMD_SW2HW_SRQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, srq_num, 0, - CMD_HW2SW_SRQ, - CMD_TIME_CLASS_A, status); -} - -int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num, - struct mthca_mailbox *mailbox, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, num, 0, - CMD_QUERY_SRQ, CMD_TIME_CLASS_A, status); -} - -int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status) -{ - return mthca_cmd(dev, limit, srq_num, 0, CMD_ARM_SRQ, - CMD_TIME_CLASS_B, status); -} - -int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, - enum ib_qp_state next, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u32 optmask, - u8 *status) -{ - static const u16 op[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = { - [IB_QPS_RESET] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_INIT] = CMD_RST2INIT_QPEE, - }, - [IB_QPS_INIT] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_INIT] = CMD_INIT2INIT_QPEE, - [IB_QPS_RTR] = CMD_INIT2RTR_QPEE, - }, - [IB_QPS_RTR] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_RTS] = CMD_RTR2RTS_QPEE, - }, - [IB_QPS_RTS] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_RTS] = CMD_RTS2RTS_QPEE, - [IB_QPS_SQD] = CMD_RTS2SQD_QPEE, - }, - [IB_QPS_SQD] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_RTS] = CMD_SQD2RTS_QPEE, - [IB_QPS_SQD] = CMD_SQD2SQD_QPEE, - }, - [IB_QPS_SQE] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - [IB_QPS_RTS] = CMD_SQERR2RTS_QPEE, - }, - [IB_QPS_ERR] = { - [IB_QPS_RESET] = CMD_ERR2RST_QPEE, - [IB_QPS_ERR] = CMD_2ERR_QPEE, - } - }; - - u8 op_mod = 0; - int my_mailbox = 0; - int err; - - if (op[cur][next] == CMD_ERR2RST_QPEE) { - op_mod = 3; /* don't write outbox, any->reset */ - - /* For debugging */ - if (!mailbox) { - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (!IS_ERR(mailbox)) { - my_mailbox = 1; - op_mod = 2; /* write outbox, any->reset */ - } else - mailbox = NULL; - } - - err = mthca_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, - (!!is_ee << 24) | num, op_mod, - op[cur][next], CMD_TIME_CLASS_C, status); - - if (0 && mailbox) { - int i; - mthca_dbg(dev, "Dumping QP context:\n"); - printk(" %08x\n", be32_to_cpup(mailbox->buf)); - for (i = 0; i < 0x100 / 4; ++i) { - if (i % 8 == 0) - printk("[%02x] ", i * 4); - printk(" %08x", - be32_to_cpu(((__be32 *) mailbox->buf)[i + 2])); - if ((i + 1) % 8 == 0) - printk("\n"); - } - } - - if (my_mailbox) - mthca_free_mailbox(dev, mailbox); - } else { - if (0) { - int i; - mthca_dbg(dev, "Dumping QP context:\n"); - printk(" opt param mask: %08x\n", be32_to_cpup(mailbox->buf)); - for (i = 0; i < 0x100 / 4; ++i) { - if (i % 8 == 0) - printk(" [%02x] ", i * 4); - printk(" %08x", - be32_to_cpu(((__be32 *) mailbox->buf)[i + 2])); - if ((i + 1) % 8 == 0) - printk("\n"); - } - } - - err = mthca_cmd(dev, mailbox->dma, optmask | (!!is_ee << 24) | num, - op_mod, op[cur][next], CMD_TIME_CLASS_C, status); - } - - return err; -} - -int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, (!!is_ee << 24) | num, 0, - CMD_QUERY_QPEE, CMD_TIME_CLASS_A, status); -} - -int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, - u8 *status) -{ - u8 op_mod; - - switch (type) { - case IB_QPT_SMI: - op_mod = 0; - break; - case IB_QPT_GSI: - op_mod = 1; - break; - case IB_QPT_RAW_IPV6: - op_mod = 2; - break; - case IB_QPT_RAW_ETHERTYPE: - op_mod = 3; - break; - default: - return -EINVAL; - } - - return mthca_cmd(dev, 0, qpn, op_mod, CMD_CONF_SPECIAL_QP, - CMD_TIME_CLASS_B, status); -} - -int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, - int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad, u8 *status) -{ - struct mthca_mailbox *inmailbox, *outmailbox; - void *inbox; - int err; - u32 in_modifier = port; - u8 op_modifier = 0; - -#define MAD_IFC_BOX_SIZE 0x400 -#define MAD_IFC_MY_QPN_OFFSET 0x100 -#define MAD_IFC_RQPN_OFFSET 0x108 -#define MAD_IFC_SL_OFFSET 0x10c -#define MAD_IFC_G_PATH_OFFSET 0x10d -#define MAD_IFC_RLID_OFFSET 0x10e -#define MAD_IFC_PKEY_OFFSET 0x112 -#define MAD_IFC_GRH_OFFSET 0x140 - - inmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(inmailbox)) - return PTR_ERR(inmailbox); - inbox = inmailbox->buf; - - outmailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(outmailbox)) { - mthca_free_mailbox(dev, inmailbox); - return PTR_ERR(outmailbox); - } - - memcpy(inbox, in_mad, 256); - - /* - * Key check traps can't be generated unless we have in_wc to - * tell us where to send the trap. - */ - if (ignore_mkey || !in_wc) - op_modifier |= 0x1; - if (ignore_bkey || !in_wc) - op_modifier |= 0x2; - - if (in_wc) { - u8 val; - - memset(inbox + 256, 0, 256); - - MTHCA_PUT(inbox, in_wc->qp->qp_num, MAD_IFC_MY_QPN_OFFSET); - MTHCA_PUT(inbox, in_wc->src_qp, MAD_IFC_RQPN_OFFSET); - - val = in_wc->sl << 4; - MTHCA_PUT(inbox, val, MAD_IFC_SL_OFFSET); - - val = in_wc->dlid_path_bits | - (in_wc->wc_flags & IB_WC_GRH ? 0x80 : 0); - MTHCA_PUT(inbox, val, MAD_IFC_G_PATH_OFFSET); - - MTHCA_PUT(inbox, in_wc->slid, MAD_IFC_RLID_OFFSET); - MTHCA_PUT(inbox, in_wc->pkey_index, MAD_IFC_PKEY_OFFSET); - - if (in_grh) - memcpy(inbox + MAD_IFC_GRH_OFFSET, in_grh, 40); - - op_modifier |= 0x4; - - in_modifier |= in_wc->slid << 16; - } - - err = mthca_cmd_box(dev, inmailbox->dma, outmailbox->dma, - in_modifier, op_modifier, - CMD_MAD_IFC, CMD_TIME_CLASS_C, status); - - if (!err && !*status) - memcpy(response_mad, outmailbox->buf, 256); - - mthca_free_mailbox(dev, inmailbox); - mthca_free_mailbox(dev, outmailbox); - return err; -} - -int mthca_READ_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, index, 0, - CMD_READ_MGM, CMD_TIME_CLASS_A, status); -} - -int mthca_WRITE_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status) -{ - return mthca_cmd(dev, mailbox->dma, index, 0, CMD_WRITE_MGM, - CMD_TIME_CLASS_A, status); -} - -int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - u16 *hash, u8 *status) -{ - u64 imm; - int err; - - err = mthca_cmd_imm(dev, mailbox->dma, &imm, 0, 0, CMD_MGID_HASH, - CMD_TIME_CLASS_A, status); - - *hash = imm; - return err; -} - -int mthca_DIAG_RPRT(struct mthca_dev *dev, int mod, - struct mthca_mailbox *mailbox, u8 *status) -{ - return mthca_cmd_box(dev, 0, mailbox->dma, 0, mod, CMD_DIAG_RPRT, - CMD_TIME_CLASS_A, status); -} - -int mthca_NOP(struct mthca_dev *dev, u8 *status) -{ - return mthca_cmd(dev, 0, 0x1f, 0, CMD_NOP, msecs_to_jiffies(100), status); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.h deleted file mode 100644 index 5dd777c43ee5..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cmd.h +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2006 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_CMD_H -#define MTHCA_CMD_H - -#include - -#define MTHCA_MAILBOX_SIZE 4096 - -enum { - /* command completed successfully: */ - MTHCA_CMD_STAT_OK = 0x00, - /* Internal error (such as a bus error) occurred while processing command: */ - MTHCA_CMD_STAT_INTERNAL_ERR = 0x01, - /* Operation/command not supported or opcode modifier not supported: */ - MTHCA_CMD_STAT_BAD_OP = 0x02, - /* Parameter not supported or parameter out of range: */ - MTHCA_CMD_STAT_BAD_PARAM = 0x03, - /* System not enabled or bad system state: */ - MTHCA_CMD_STAT_BAD_SYS_STATE = 0x04, - /* Attempt to access reserved or unallocaterd resource: */ - MTHCA_CMD_STAT_BAD_RESOURCE = 0x05, - /* Requested resource is currently executing a command, or is otherwise busy: */ - MTHCA_CMD_STAT_RESOURCE_BUSY = 0x06, - /* memory error: */ - MTHCA_CMD_STAT_DDR_MEM_ERR = 0x07, - /* Required capability exceeds device limits: */ - MTHCA_CMD_STAT_EXCEED_LIM = 0x08, - /* Resource is not in the appropriate state or ownership: */ - MTHCA_CMD_STAT_BAD_RES_STATE = 0x09, - /* Index out of range: */ - MTHCA_CMD_STAT_BAD_INDEX = 0x0a, - /* FW image corrupted: */ - MTHCA_CMD_STAT_BAD_NVMEM = 0x0b, - /* Attempt to modify a QP/EE which is not in the presumed state: */ - MTHCA_CMD_STAT_BAD_QPEE_STATE = 0x10, - /* Bad segment parameters (Address/Size): */ - MTHCA_CMD_STAT_BAD_SEG_PARAM = 0x20, - /* Memory Region has Memory Windows bound to: */ - MTHCA_CMD_STAT_REG_BOUND = 0x21, - /* HCA local attached memory not present: */ - MTHCA_CMD_STAT_LAM_NOT_PRE = 0x22, - /* Bad management packet (silently discarded): */ - MTHCA_CMD_STAT_BAD_PKT = 0x30, - /* More outstanding CQEs in CQ than new CQ size: */ - MTHCA_CMD_STAT_BAD_SIZE = 0x40 -}; - -enum { - MTHCA_TRANS_INVALID = 0, - MTHCA_TRANS_RST2INIT, - MTHCA_TRANS_INIT2INIT, - MTHCA_TRANS_INIT2RTR, - MTHCA_TRANS_RTR2RTS, - MTHCA_TRANS_RTS2RTS, - MTHCA_TRANS_SQERR2RTS, - MTHCA_TRANS_ANY2ERR, - MTHCA_TRANS_RTS2SQD, - MTHCA_TRANS_SQD2SQD, - MTHCA_TRANS_SQD2RTS, - MTHCA_TRANS_ANY2RST, -}; - -enum { - DEV_LIM_FLAG_RC = 1 << 0, - DEV_LIM_FLAG_UC = 1 << 1, - DEV_LIM_FLAG_UD = 1 << 2, - DEV_LIM_FLAG_RD = 1 << 3, - DEV_LIM_FLAG_RAW_IPV6 = 1 << 4, - DEV_LIM_FLAG_RAW_ETHER = 1 << 5, - DEV_LIM_FLAG_SRQ = 1 << 6, - DEV_LIM_FLAG_IPOIB_CSUM = 1 << 7, - DEV_LIM_FLAG_BAD_PKEY_CNTR = 1 << 8, - DEV_LIM_FLAG_BAD_QKEY_CNTR = 1 << 9, - DEV_LIM_FLAG_MW = 1 << 16, - DEV_LIM_FLAG_AUTO_PATH_MIG = 1 << 17, - DEV_LIM_FLAG_ATOMIC = 1 << 18, - DEV_LIM_FLAG_RAW_MULTI = 1 << 19, - DEV_LIM_FLAG_UD_AV_PORT_ENFORCE = 1 << 20, - DEV_LIM_FLAG_UD_MULTI = 1 << 21, -}; - -enum { - DIAG_RPRT_Q_XPRT_CIERR = 2, - DIAG_RPRT_QR_XPRT_CIERR = 3, - DIAG_RPRT_Q_PERF = 4, - DIAG_RPRT_QR_PERF = 5, - DIAG_RPRT_Q_MISC = 6, - DIAG_RPRT_QR_MISC = 7, -}; - -struct mthca_mailbox { - dma_addr_t dma; - void *buf; -}; - -struct mthca_dev_lim { - int max_srq_sz; - int max_qp_sz; - int reserved_qps; - int max_qps; - int reserved_srqs; - int max_srqs; - int reserved_eecs; - int max_eecs; - int max_cq_sz; - int reserved_cqs; - int max_cqs; - int max_mpts; - int reserved_eqs; - int max_eqs; - int reserved_mtts; - int max_mrw_sz; - int reserved_mrws; - int max_mtt_seg; - int max_requester_per_qp; - int max_responder_per_qp; - int max_rdma_global; - int local_ca_ack_delay; - int max_mtu; - int max_port_width; - int max_vl; - int num_ports; - int max_gids; - u16 stat_rate_support; - int max_pkeys; - u32 flags; - int reserved_uars; - int uar_size; - int min_page_sz; - int max_sg; - int max_desc_sz; - int max_qp_per_mcg; - int reserved_mgms; - int max_mcgs; - int reserved_pds; - int max_pds; - int reserved_rdds; - int max_rdds; - int eec_entry_sz; - int qpc_entry_sz; - int eeec_entry_sz; - int eqpc_entry_sz; - int eqc_entry_sz; - int cqc_entry_sz; - int srq_entry_sz; - int uar_scratch_entry_sz; - int mpt_entry_sz; - union { - struct { - int max_avs; - } tavor; - struct { - int resize_srq; - int max_pbl_sz; - u8 bmme_flags; - u32 reserved_lkey; - int lam_required; - u64 max_icm_sz; - } arbel; - } hca; -}; - -struct mthca_adapter { - u32 vendor_id; - u32 device_id; - u32 revision_id; - char board_id[MTHCA_BOARD_ID_LEN]; - u8 inta_pin; -}; - -struct mthca_init_hca_param { - u64 qpc_base; - u64 eec_base; - u64 srqc_base; - u64 cqc_base; - u64 eqpc_base; - u64 eeec_base; - u64 eqc_base; - u64 rdb_base; - u64 mc_base; - u64 mpt_base; - u64 mtt_base; - u64 uar_scratch_base; - u64 uarc_base; - u16 log_mc_entry_sz; - u16 mc_hash_sz; - u8 log_num_qps; - u8 log_num_eecs; - u8 log_num_srqs; - u8 log_num_cqs; - u8 log_num_eqs; - u8 log_mc_table_sz; - u8 mtt_seg_sz; - u8 log_mpt_sz; - u8 log_uar_sz; - u8 log_uarc_sz; -}; - -struct mthca_init_ib_param { - int port_width; - int vl_cap; - int mtu_cap; - u16 gid_cap; - u16 pkey_cap; - int set_guid0; - u64 guid0; - int set_node_guid; - u64 node_guid; - int set_si_guid; - u64 si_guid; -}; - -struct mthca_set_ib_param { - int set_si_guid; - int reset_qkey_viol; - u64 si_guid; - u32 cap_mask; -}; - -int mthca_cmd_init(struct mthca_dev *dev); -void mthca_cmd_cleanup(struct mthca_dev *dev); -int mthca_cmd_use_events(struct mthca_dev *dev); -void mthca_cmd_use_polling(struct mthca_dev *dev); -void mthca_cmd_event(struct mthca_dev *dev, u16 token, - u8 status, u64 out_param); - -struct mthca_mailbox *mthca_alloc_mailbox(struct mthca_dev *dev, - gfp_t gfp_mask); -void mthca_free_mailbox(struct mthca_dev *dev, struct mthca_mailbox *mailbox); - -int mthca_SYS_EN(struct mthca_dev *dev, u8 *status); -int mthca_SYS_DIS(struct mthca_dev *dev, u8 *status); -int mthca_MAP_FA(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status); -int mthca_UNMAP_FA(struct mthca_dev *dev, u8 *status); -int mthca_RUN_FW(struct mthca_dev *dev, u8 *status); -int mthca_QUERY_FW(struct mthca_dev *dev, u8 *status); -int mthca_ENABLE_LAM(struct mthca_dev *dev, u8 *status); -int mthca_DISABLE_LAM(struct mthca_dev *dev, u8 *status); -int mthca_QUERY_DDR(struct mthca_dev *dev, u8 *status); -int mthca_QUERY_DEV_LIM(struct mthca_dev *dev, - struct mthca_dev_lim *dev_lim, u8 *status); -int mthca_QUERY_ADAPTER(struct mthca_dev *dev, - struct mthca_adapter *adapter, u8 *status); -int mthca_INIT_HCA(struct mthca_dev *dev, - struct mthca_init_hca_param *param, - u8 *status); -int mthca_INIT_IB(struct mthca_dev *dev, - struct mthca_init_ib_param *param, - int port, u8 *status); -int mthca_CLOSE_IB(struct mthca_dev *dev, int port, u8 *status); -int mthca_CLOSE_HCA(struct mthca_dev *dev, int panic, u8 *status); -int mthca_SET_IB(struct mthca_dev *dev, struct mthca_set_ib_param *param, - int port, u8 *status); -int mthca_MAP_ICM(struct mthca_dev *dev, struct mthca_icm *icm, u64 virt, u8 *status); -int mthca_MAP_ICM_page(struct mthca_dev *dev, u64 dma_addr, u64 virt, u8 *status); -int mthca_UNMAP_ICM(struct mthca_dev *dev, u64 virt, u32 page_count, u8 *status); -int mthca_MAP_ICM_AUX(struct mthca_dev *dev, struct mthca_icm *icm, u8 *status); -int mthca_UNMAP_ICM_AUX(struct mthca_dev *dev, u8 *status); -int mthca_SET_ICM_SIZE(struct mthca_dev *dev, u64 icm_size, u64 *aux_pages, - u8 *status); -int mthca_SW2HW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status); -int mthca_HW2SW_MPT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int mpt_index, u8 *status); -int mthca_WRITE_MTT(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int num_mtt, u8 *status); -int mthca_SYNC_TPT(struct mthca_dev *dev, u8 *status); -int mthca_MAP_EQ(struct mthca_dev *dev, u64 event_mask, int unmap, - int eq_num, u8 *status); -int mthca_SW2HW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status); -int mthca_HW2SW_EQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int eq_num, u8 *status); -int mthca_SW2HW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status); -int mthca_HW2SW_CQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int cq_num, u8 *status); -int mthca_RESIZE_CQ(struct mthca_dev *dev, int cq_num, u32 lkey, u8 log_size, - u8 *status); -int mthca_SW2HW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status); -int mthca_HW2SW_SRQ(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - int srq_num, u8 *status); -int mthca_QUERY_SRQ(struct mthca_dev *dev, u32 num, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_ARM_SRQ(struct mthca_dev *dev, int srq_num, int limit, u8 *status); -int mthca_MODIFY_QP(struct mthca_dev *dev, enum ib_qp_state cur, - enum ib_qp_state next, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u32 optmask, - u8 *status); -int mthca_QUERY_QP(struct mthca_dev *dev, u32 num, int is_ee, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_CONF_SPECIAL_QP(struct mthca_dev *dev, int type, u32 qpn, - u8 *status); -int mthca_MAD_IFC(struct mthca_dev *dev, int ignore_mkey, int ignore_bkey, - int port, struct ib_wc *in_wc, struct ib_grh *in_grh, - void *in_mad, void *response_mad, u8 *status); -int mthca_READ_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_WRITE_MGM(struct mthca_dev *dev, int index, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_MGID_HASH(struct mthca_dev *dev, struct mthca_mailbox *mailbox, - u16 *hash, u8 *status); -int mthca_DIAG_RPRT(struct mthca_dev *dev, int mod, - struct mthca_mailbox *mailbox, u8 *status); -int mthca_NOP(struct mthca_dev *dev, u8 *status); - -#endif /* MTHCA_CMD_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_config_reg.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_config_reg.h deleted file mode 100644 index 43c08943668c..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_config_reg.h +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_CONFIG_REG_H -#define MTHCA_CONFIG_REG_H - -#include - -#define MTHCA_HCR_BASE 0x80680 -#define MTHCA_HCR_SIZE 0x0001c -#define MTHCA_ECR_BASE 0x80700 -#define MTHCA_ECR_SIZE 0x00008 -#define MTHCA_ECR_CLR_BASE 0x80708 -#define MTHCA_ECR_CLR_SIZE 0x00008 -#define MTHCA_MAP_ECR_SIZE (MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE) -#define MTHCA_CLR_INT_BASE 0xf00d8 -#define MTHCA_CLR_INT_SIZE 0x00008 -#define MTHCA_EQ_SET_CI_SIZE (8 * 32) - -#endif /* MTHCA_CONFIG_REG_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cq.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_cq.c deleted file mode 100644 index aa75d26ac9ab..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_cq.c +++ /dev/null @@ -1,992 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include - -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_memfree.h" - -enum { - MTHCA_MAX_DIRECT_CQ_SIZE = 4 * PAGE_SIZE -}; - -enum { - MTHCA_CQ_ENTRY_SIZE = 0x20 -}; - -enum { - MTHCA_ATOMIC_BYTE_LEN = 8 -}; - -/* - * Must be packed because start is 64 bits but only aligned to 32 bits. - */ -struct mthca_cq_context { - __be32 flags; - __be64 start; - __be32 logsize_usrpage; - __be32 error_eqn; /* Tavor only */ - __be32 comp_eqn; - __be32 pd; - __be32 lkey; - __be32 last_notified_index; - __be32 solicit_producer_index; - __be32 consumer_index; - __be32 producer_index; - __be32 cqn; - __be32 ci_db; /* Arbel only */ - __be32 state_db; /* Arbel only */ - u32 reserved; -} __attribute__((packed)); - -#define MTHCA_CQ_STATUS_OK ( 0 << 28) -#define MTHCA_CQ_STATUS_OVERFLOW ( 9 << 28) -#define MTHCA_CQ_STATUS_WRITE_FAIL (10 << 28) -#define MTHCA_CQ_FLAG_TR ( 1 << 18) -#define MTHCA_CQ_FLAG_OI ( 1 << 17) -#define MTHCA_CQ_STATE_DISARMED ( 0 << 8) -#define MTHCA_CQ_STATE_ARMED ( 1 << 8) -#define MTHCA_CQ_STATE_ARMED_SOL ( 4 << 8) -#define MTHCA_EQ_STATE_FIRED (10 << 8) - -enum { - MTHCA_ERROR_CQE_OPCODE_MASK = 0xfe -}; - -enum { - SYNDROME_LOCAL_LENGTH_ERR = 0x01, - SYNDROME_LOCAL_QP_OP_ERR = 0x02, - SYNDROME_LOCAL_EEC_OP_ERR = 0x03, - SYNDROME_LOCAL_PROT_ERR = 0x04, - SYNDROME_WR_FLUSH_ERR = 0x05, - SYNDROME_MW_BIND_ERR = 0x06, - SYNDROME_BAD_RESP_ERR = 0x10, - SYNDROME_LOCAL_ACCESS_ERR = 0x11, - SYNDROME_REMOTE_INVAL_REQ_ERR = 0x12, - SYNDROME_REMOTE_ACCESS_ERR = 0x13, - SYNDROME_REMOTE_OP_ERR = 0x14, - SYNDROME_RETRY_EXC_ERR = 0x15, - SYNDROME_RNR_RETRY_EXC_ERR = 0x16, - SYNDROME_LOCAL_RDD_VIOL_ERR = 0x20, - SYNDROME_REMOTE_INVAL_RD_REQ_ERR = 0x21, - SYNDROME_REMOTE_ABORTED_ERR = 0x22, - SYNDROME_INVAL_EECN_ERR = 0x23, - SYNDROME_INVAL_EEC_STATE_ERR = 0x24 -}; - -struct mthca_cqe { - __be32 my_qpn; - __be32 my_ee; - __be32 rqpn; - u8 sl_ipok; - u8 g_mlpath; - __be16 rlid; - __be32 imm_etype_pkey_eec; - __be32 byte_cnt; - __be32 wqe; - u8 opcode; - u8 is_send; - u8 reserved; - u8 owner; -}; - -struct mthca_err_cqe { - __be32 my_qpn; - u32 reserved1[3]; - u8 syndrome; - u8 vendor_err; - __be16 db_cnt; - u32 reserved2; - __be32 wqe; - u8 opcode; - u8 reserved3[2]; - u8 owner; -}; - -#define MTHCA_CQ_ENTRY_OWNER_SW (0 << 7) -#define MTHCA_CQ_ENTRY_OWNER_HW (1 << 7) - -#define MTHCA_TAVOR_CQ_DB_INC_CI (1 << 24) -#define MTHCA_TAVOR_CQ_DB_REQ_NOT (2 << 24) -#define MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL (3 << 24) -#define MTHCA_TAVOR_CQ_DB_SET_CI (4 << 24) -#define MTHCA_TAVOR_CQ_DB_REQ_NOT_MULT (5 << 24) - -#define MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL (1 << 24) -#define MTHCA_ARBEL_CQ_DB_REQ_NOT (2 << 24) -#define MTHCA_ARBEL_CQ_DB_REQ_NOT_MULT (3 << 24) - -static inline struct mthca_cqe *get_cqe_from_buf(struct mthca_cq_buf *buf, - int entry) -{ - if (buf->is_direct) - return buf->queue.direct.buf + (entry * MTHCA_CQ_ENTRY_SIZE); - else - return buf->queue.page_list[entry * MTHCA_CQ_ENTRY_SIZE / PAGE_SIZE].buf - + (entry * MTHCA_CQ_ENTRY_SIZE) % PAGE_SIZE; -} - -static inline struct mthca_cqe *get_cqe(struct mthca_cq *cq, int entry) -{ - return get_cqe_from_buf(&cq->buf, entry); -} - -static inline struct mthca_cqe *cqe_sw(struct mthca_cqe *cqe) -{ - return MTHCA_CQ_ENTRY_OWNER_HW & cqe->owner ? NULL : cqe; -} - -static inline struct mthca_cqe *next_cqe_sw(struct mthca_cq *cq) -{ - return cqe_sw(get_cqe(cq, cq->cons_index & cq->ibcq.cqe)); -} - -static inline void set_cqe_hw(struct mthca_cqe *cqe) -{ - cqe->owner = MTHCA_CQ_ENTRY_OWNER_HW; -} - -static void dump_cqe(struct mthca_dev *dev, void *cqe_ptr) -{ - __be32 *cqe = cqe_ptr; - - (void) cqe; /* avoid warning if mthca_dbg compiled away... */ - mthca_dbg(dev, "CQE contents %08x %08x %08x %08x %08x %08x %08x %08x\n", - be32_to_cpu(cqe[0]), be32_to_cpu(cqe[1]), be32_to_cpu(cqe[2]), - be32_to_cpu(cqe[3]), be32_to_cpu(cqe[4]), be32_to_cpu(cqe[5]), - be32_to_cpu(cqe[6]), be32_to_cpu(cqe[7])); -} - -/* - * incr is ignored in native Arbel (mem-free) mode, so cq->cons_index - * should be correct before calling update_cons_index(). - */ -static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq, - int incr) -{ - if (mthca_is_memfree(dev)) { - *cq->set_ci_db = cpu_to_be32(cq->cons_index); - wmb(); - } else { - mthca_write64(MTHCA_TAVOR_CQ_DB_INC_CI | cq->cqn, incr - 1, - dev->kar + MTHCA_CQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - /* - * Make sure doorbells don't leak out of CQ spinlock - * and reach the HCA out of order: - */ - mmiowb(); - } -} - -void mthca_cq_completion(struct mthca_dev *dev, u32 cqn) -{ - struct mthca_cq *cq; - - cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); - - if (!cq) { - mthca_warn(dev, "Completion event for bogus CQ %08x\n", cqn); - return; - } - - ++cq->arm_sn; - - cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); -} - -void mthca_cq_event(struct mthca_dev *dev, u32 cqn, - enum ib_event_type event_type) -{ - struct mthca_cq *cq; - struct ib_event event; - - spin_lock(&dev->cq_table.lock); - - cq = mthca_array_get(&dev->cq_table.cq, cqn & (dev->limits.num_cqs - 1)); - if (cq) - ++cq->refcount; - - spin_unlock(&dev->cq_table.lock); - - if (!cq) { - mthca_warn(dev, "Async event for bogus CQ %08x\n", cqn); - return; - } - - event.device = &dev->ib_dev; - event.event = event_type; - event.element.cq = &cq->ibcq; - if (cq->ibcq.event_handler) - cq->ibcq.event_handler(&event, cq->ibcq.cq_context); - - spin_lock(&dev->cq_table.lock); - if (!--cq->refcount) - wake_up(&cq->wait); - spin_unlock(&dev->cq_table.lock); -} - -static inline int is_recv_cqe(struct mthca_cqe *cqe) -{ - if ((cqe->opcode & MTHCA_ERROR_CQE_OPCODE_MASK) == - MTHCA_ERROR_CQE_OPCODE_MASK) - return !(cqe->opcode & 0x01); - else - return !(cqe->is_send & 0x80); -} - -void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, - struct mthca_srq *srq) -{ - struct mthca_cqe *cqe; - u32 prod_index; - int i, nfreed = 0; - - spin_lock_irq(&cq->lock); - - /* - * First we need to find the current producer index, so we - * know where to start cleaning from. It doesn't matter if HW - * adds new entries after this loop -- the QP we're worried - * about is already in RESET, so the new entries won't come - * from our QP and therefore don't need to be checked. - */ - for (prod_index = cq->cons_index; - cqe_sw(get_cqe(cq, prod_index & cq->ibcq.cqe)); - ++prod_index) - if (prod_index == cq->cons_index + cq->ibcq.cqe) - break; - - if (0) - mthca_dbg(dev, "Cleaning QPN %06x from CQN %06x; ci %d, pi %d\n", - qpn, cq->cqn, cq->cons_index, prod_index); - - /* - * Now sweep backwards through the CQ, removing CQ entries - * that match our QP by copying older entries on top of them. - */ - while ((int) --prod_index - (int) cq->cons_index >= 0) { - cqe = get_cqe(cq, prod_index & cq->ibcq.cqe); - if (cqe->my_qpn == cpu_to_be32(qpn)) { - if (srq && is_recv_cqe(cqe)) - mthca_free_srq_wqe(srq, be32_to_cpu(cqe->wqe)); - ++nfreed; - } else if (nfreed) - memcpy(get_cqe(cq, (prod_index + nfreed) & cq->ibcq.cqe), - cqe, MTHCA_CQ_ENTRY_SIZE); - } - - if (nfreed) { - for (i = 0; i < nfreed; ++i) - set_cqe_hw(get_cqe(cq, (cq->cons_index + i) & cq->ibcq.cqe)); - wmb(); - cq->cons_index += nfreed; - update_cons_index(dev, cq, nfreed); - } - - spin_unlock_irq(&cq->lock); -} - -void mthca_cq_resize_copy_cqes(struct mthca_cq *cq) -{ - int i; - - /* - * In Tavor mode, the hardware keeps the consumer and producer - * indices mod the CQ size. Since we might be making the CQ - * bigger, we need to deal with the case where the producer - * index wrapped around before the CQ was resized. - */ - if (!mthca_is_memfree(to_mdev(cq->ibcq.device)) && - cq->ibcq.cqe < cq->resize_buf->cqe) { - cq->cons_index &= cq->ibcq.cqe; - if (cqe_sw(get_cqe(cq, cq->ibcq.cqe))) - cq->cons_index -= cq->ibcq.cqe + 1; - } - - for (i = cq->cons_index; cqe_sw(get_cqe(cq, i & cq->ibcq.cqe)); ++i) - memcpy(get_cqe_from_buf(&cq->resize_buf->buf, - i & cq->resize_buf->cqe), - get_cqe(cq, i & cq->ibcq.cqe), MTHCA_CQ_ENTRY_SIZE); -} - -int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent) -{ - int ret; - int i; - - ret = mthca_buf_alloc(dev, nent * MTHCA_CQ_ENTRY_SIZE, - MTHCA_MAX_DIRECT_CQ_SIZE, - &buf->queue, &buf->is_direct, - &dev->driver_pd, 1, &buf->mr); - if (ret) - return ret; - - for (i = 0; i < nent; ++i) - set_cqe_hw(get_cqe_from_buf(buf, i)); - - return 0; -} - -void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe) -{ - mthca_buf_free(dev, (cqe + 1) * MTHCA_CQ_ENTRY_SIZE, &buf->queue, - buf->is_direct, &buf->mr); -} - -static void handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq, - struct mthca_qp *qp, int wqe_index, int is_send, - struct mthca_err_cqe *cqe, - struct ib_wc *entry, int *free_cqe) -{ - int dbd; - __be32 new_wqe; - - if (cqe->syndrome == SYNDROME_LOCAL_QP_OP_ERR) { - mthca_dbg(dev, "local QP operation err " - "(QPN %06x, WQE @ %08x, CQN %06x, index %d)\n", - be32_to_cpu(cqe->my_qpn), be32_to_cpu(cqe->wqe), - cq->cqn, cq->cons_index); - dump_cqe(dev, cqe); - } - - /* - * For completions in error, only work request ID, status, vendor error - * (and freed resource count for RD) have to be set. - */ - switch (cqe->syndrome) { - case SYNDROME_LOCAL_LENGTH_ERR: - entry->status = IB_WC_LOC_LEN_ERR; - break; - case SYNDROME_LOCAL_QP_OP_ERR: - entry->status = IB_WC_LOC_QP_OP_ERR; - break; - case SYNDROME_LOCAL_EEC_OP_ERR: - entry->status = IB_WC_LOC_EEC_OP_ERR; - break; - case SYNDROME_LOCAL_PROT_ERR: - entry->status = IB_WC_LOC_PROT_ERR; - break; - case SYNDROME_WR_FLUSH_ERR: - entry->status = IB_WC_WR_FLUSH_ERR; - break; - case SYNDROME_MW_BIND_ERR: - entry->status = IB_WC_MW_BIND_ERR; - break; - case SYNDROME_BAD_RESP_ERR: - entry->status = IB_WC_BAD_RESP_ERR; - break; - case SYNDROME_LOCAL_ACCESS_ERR: - entry->status = IB_WC_LOC_ACCESS_ERR; - break; - case SYNDROME_REMOTE_INVAL_REQ_ERR: - entry->status = IB_WC_REM_INV_REQ_ERR; - break; - case SYNDROME_REMOTE_ACCESS_ERR: - entry->status = IB_WC_REM_ACCESS_ERR; - break; - case SYNDROME_REMOTE_OP_ERR: - entry->status = IB_WC_REM_OP_ERR; - break; - case SYNDROME_RETRY_EXC_ERR: - entry->status = IB_WC_RETRY_EXC_ERR; - break; - case SYNDROME_RNR_RETRY_EXC_ERR: - entry->status = IB_WC_RNR_RETRY_EXC_ERR; - break; - case SYNDROME_LOCAL_RDD_VIOL_ERR: - entry->status = IB_WC_LOC_RDD_VIOL_ERR; - break; - case SYNDROME_REMOTE_INVAL_RD_REQ_ERR: - entry->status = IB_WC_REM_INV_RD_REQ_ERR; - break; - case SYNDROME_REMOTE_ABORTED_ERR: - entry->status = IB_WC_REM_ABORT_ERR; - break; - case SYNDROME_INVAL_EECN_ERR: - entry->status = IB_WC_INV_EECN_ERR; - break; - case SYNDROME_INVAL_EEC_STATE_ERR: - entry->status = IB_WC_INV_EEC_STATE_ERR; - break; - default: - entry->status = IB_WC_GENERAL_ERR; - break; - } - - entry->vendor_err = cqe->vendor_err; - - /* - * Mem-free HCAs always generate one CQE per WQE, even in the - * error case, so we don't have to check the doorbell count, etc. - */ - if (mthca_is_memfree(dev)) - return; - - mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe); - - /* - * If we're at the end of the WQE chain, or we've used up our - * doorbell count, free the CQE. Otherwise just update it for - * the next poll operation. - */ - if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd)) - return; - - be16_add_cpu(&cqe->db_cnt, -dbd); - cqe->wqe = new_wqe; - cqe->syndrome = SYNDROME_WR_FLUSH_ERR; - - *free_cqe = 0; -} - -static inline int mthca_poll_one(struct mthca_dev *dev, - struct mthca_cq *cq, - struct mthca_qp **cur_qp, - int *freed, - struct ib_wc *entry) -{ - struct mthca_wq *wq; - struct mthca_cqe *cqe; - int wqe_index; - int is_error; - int is_send; - int free_cqe = 1; - int err = 0; - u16 checksum; - - cqe = next_cqe_sw(cq); - if (!cqe) - return -EAGAIN; - - /* - * Make sure we read CQ entry contents after we've checked the - * ownership bit. - */ - rmb(); - - if (0) { - mthca_dbg(dev, "%x/%d: CQE -> QPN %06x, WQE @ %08x\n", - cq->cqn, cq->cons_index, be32_to_cpu(cqe->my_qpn), - be32_to_cpu(cqe->wqe)); - dump_cqe(dev, cqe); - } - - is_error = (cqe->opcode & MTHCA_ERROR_CQE_OPCODE_MASK) == - MTHCA_ERROR_CQE_OPCODE_MASK; - is_send = is_error ? cqe->opcode & 0x01 : cqe->is_send & 0x80; - - if (!*cur_qp || be32_to_cpu(cqe->my_qpn) != (*cur_qp)->qpn) { - /* - * We do not have to take the QP table lock here, - * because CQs will be locked while QPs are removed - * from the table. - */ - *cur_qp = mthca_array_get(&dev->qp_table.qp, - be32_to_cpu(cqe->my_qpn) & - (dev->limits.num_qps - 1)); - if (!*cur_qp) { - mthca_warn(dev, "CQ entry for unknown QP %06x\n", - be32_to_cpu(cqe->my_qpn) & 0xffffff); - err = -EINVAL; - goto out; - } - } - - entry->qp = &(*cur_qp)->ibqp; - - if (is_send) { - wq = &(*cur_qp)->sq; - wqe_index = ((be32_to_cpu(cqe->wqe) - (*cur_qp)->send_wqe_offset) - >> wq->wqe_shift); - entry->wr_id = (*cur_qp)->wrid[wqe_index]; - } else if ((*cur_qp)->ibqp.srq) { - struct mthca_srq *srq = to_msrq((*cur_qp)->ibqp.srq); - u32 wqe = be32_to_cpu(cqe->wqe); - wq = NULL; - wqe_index = wqe >> srq->wqe_shift; - entry->wr_id = srq->wrid[wqe_index]; - mthca_free_srq_wqe(srq, wqe); - } else { - s32 wqe; - wq = &(*cur_qp)->rq; - wqe = be32_to_cpu(cqe->wqe); - wqe_index = wqe >> wq->wqe_shift; - /* - * WQE addr == base - 1 might be reported in receive completion - * with error instead of (rq size - 1) by Sinai FW 1.0.800 and - * Arbel FW 5.1.400. This bug should be fixed in later FW revs. - */ - if (unlikely(wqe_index < 0)) - wqe_index = wq->max - 1; - entry->wr_id = (*cur_qp)->wrid[wqe_index + (*cur_qp)->sq.max]; - } - - if (wq) { - if (wq->last_comp < wqe_index) - wq->tail += wqe_index - wq->last_comp; - else - wq->tail += wqe_index + wq->max - wq->last_comp; - - wq->last_comp = wqe_index; - } - - if (is_error) { - handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send, - (struct mthca_err_cqe *) cqe, - entry, &free_cqe); - goto out; - } - - if (is_send) { - entry->wc_flags = 0; - switch (cqe->opcode) { - case MTHCA_OPCODE_RDMA_WRITE: - entry->opcode = IB_WC_RDMA_WRITE; - break; - case MTHCA_OPCODE_RDMA_WRITE_IMM: - entry->opcode = IB_WC_RDMA_WRITE; - entry->wc_flags |= IB_WC_WITH_IMM; - break; - case MTHCA_OPCODE_SEND: - entry->opcode = IB_WC_SEND; - break; - case MTHCA_OPCODE_SEND_IMM: - entry->opcode = IB_WC_SEND; - entry->wc_flags |= IB_WC_WITH_IMM; - break; - case MTHCA_OPCODE_RDMA_READ: - entry->opcode = IB_WC_RDMA_READ; - entry->byte_len = be32_to_cpu(cqe->byte_cnt); - break; - case MTHCA_OPCODE_ATOMIC_CS: - entry->opcode = IB_WC_COMP_SWAP; - entry->byte_len = MTHCA_ATOMIC_BYTE_LEN; - break; - case MTHCA_OPCODE_ATOMIC_FA: - entry->opcode = IB_WC_FETCH_ADD; - entry->byte_len = MTHCA_ATOMIC_BYTE_LEN; - break; - case MTHCA_OPCODE_BIND_MW: - entry->opcode = IB_WC_BIND_MW; - break; - default: - entry->opcode = MTHCA_OPCODE_INVALID; - break; - } - } else { - entry->byte_len = be32_to_cpu(cqe->byte_cnt); - switch (cqe->opcode & 0x1f) { - case IB_OPCODE_SEND_LAST_WITH_IMMEDIATE: - case IB_OPCODE_SEND_ONLY_WITH_IMMEDIATE: - entry->wc_flags = IB_WC_WITH_IMM; - entry->ex.imm_data = cqe->imm_etype_pkey_eec; - entry->opcode = IB_WC_RECV; - break; - case IB_OPCODE_RDMA_WRITE_LAST_WITH_IMMEDIATE: - case IB_OPCODE_RDMA_WRITE_ONLY_WITH_IMMEDIATE: - entry->wc_flags = IB_WC_WITH_IMM; - entry->ex.imm_data = cqe->imm_etype_pkey_eec; - entry->opcode = IB_WC_RECV_RDMA_WITH_IMM; - break; - default: - entry->wc_flags = 0; - entry->opcode = IB_WC_RECV; - break; - } - entry->slid = be16_to_cpu(cqe->rlid); - entry->sl = cqe->sl_ipok >> 4; - entry->src_qp = be32_to_cpu(cqe->rqpn) & 0xffffff; - entry->dlid_path_bits = cqe->g_mlpath & 0x7f; - entry->pkey_index = be32_to_cpu(cqe->imm_etype_pkey_eec) >> 16; - entry->wc_flags |= cqe->g_mlpath & 0x80 ? IB_WC_GRH : 0; - checksum = (be32_to_cpu(cqe->rqpn) >> 24) | - ((be32_to_cpu(cqe->my_ee) >> 16) & 0xff00); - entry->csum_ok = (cqe->sl_ipok & 1 && checksum == 0xffff); - } - - entry->status = IB_WC_SUCCESS; - - out: - if (likely(free_cqe)) { - set_cqe_hw(cqe); - ++(*freed); - ++cq->cons_index; - } - - return err; -} - -int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, - struct ib_wc *entry) -{ - struct mthca_dev *dev = to_mdev(ibcq->device); - struct mthca_cq *cq = to_mcq(ibcq); - struct mthca_qp *qp = NULL; - unsigned long flags; - int err = 0; - int freed = 0; - int npolled; - - spin_lock_irqsave(&cq->lock, flags); - - npolled = 0; -repoll: - while (npolled < num_entries) { - err = mthca_poll_one(dev, cq, &qp, - &freed, entry + npolled); - if (err) - break; - ++npolled; - } - - if (freed) { - wmb(); - update_cons_index(dev, cq, freed); - } - - /* - * If a CQ resize is in progress and we discovered that the - * old buffer is empty, then peek in the new buffer, and if - * it's not empty, switch to the new buffer and continue - * polling there. - */ - if (unlikely(err == -EAGAIN && cq->resize_buf && - cq->resize_buf->state == CQ_RESIZE_READY)) { - /* - * In Tavor mode, the hardware keeps the producer - * index modulo the CQ size. Since we might be making - * the CQ bigger, we need to mask our consumer index - * using the size of the old CQ buffer before looking - * in the new CQ buffer. - */ - if (!mthca_is_memfree(dev)) - cq->cons_index &= cq->ibcq.cqe; - - if (cqe_sw(get_cqe_from_buf(&cq->resize_buf->buf, - cq->cons_index & cq->resize_buf->cqe))) { - struct mthca_cq_buf tbuf; - int tcqe; - - tbuf = cq->buf; - tcqe = cq->ibcq.cqe; - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - - cq->resize_buf->buf = tbuf; - cq->resize_buf->cqe = tcqe; - cq->resize_buf->state = CQ_RESIZE_SWAPPED; - - goto repoll; - } - } - - spin_unlock_irqrestore(&cq->lock, flags); - - return err == 0 || err == -EAGAIN ? npolled : err; -} - -int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags) -{ - u32 dbhi = ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? - MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL : - MTHCA_TAVOR_CQ_DB_REQ_NOT) | - to_mcq(cq)->cqn; - - mthca_write64(dbhi, 0xffffffff, to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&to_mdev(cq->device)->doorbell_lock)); - - return 0; -} - -int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) -{ - struct mthca_cq *cq = to_mcq(ibcq); - __be32 db_rec[2]; - u32 dbhi; - u32 sn = cq->arm_sn & 3; - - db_rec[0] = cpu_to_be32(cq->cons_index); - db_rec[1] = cpu_to_be32((cq->cqn << 8) | (2 << 5) | (sn << 3) | - ((flags & IB_CQ_SOLICITED_MASK) == - IB_CQ_SOLICITED ? 1 : 2)); - - mthca_write_db_rec(db_rec, cq->arm_db); - - /* - * Make sure that the doorbell record in host memory is - * written before ringing the doorbell via PCI MMIO. - */ - wmb(); - - dbhi = (sn << 28) | - ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED ? - MTHCA_ARBEL_CQ_DB_REQ_NOT_SOL : - MTHCA_ARBEL_CQ_DB_REQ_NOT) | cq->cqn; - - mthca_write64(dbhi, cq->cons_index, - to_mdev(ibcq->device)->kar + MTHCA_CQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&to_mdev(ibcq->device)->doorbell_lock)); - - return 0; -} - -int mthca_init_cq(struct mthca_dev *dev, int nent, - struct mthca_ucontext *ctx, u32 pdn, - struct mthca_cq *cq) -{ - struct mthca_mailbox *mailbox; - struct mthca_cq_context *cq_context; - int err = -ENOMEM; - u8 status; - - cq->ibcq.cqe = nent - 1; - cq->is_kernel = !ctx; - - cq->cqn = mthca_alloc(&dev->cq_table.alloc); - if (cq->cqn == -1) - return -ENOMEM; - - if (mthca_is_memfree(dev)) { - err = mthca_table_get(dev, dev->cq_table.table, cq->cqn); - if (err) - goto err_out; - - if (cq->is_kernel) { - cq->arm_sn = 1; - - err = -ENOMEM; - - cq->set_ci_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, - cq->cqn, &cq->set_ci_db); - if (cq->set_ci_db_index < 0) - goto err_out_icm; - - cq->arm_db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_CQ_ARM, - cq->cqn, &cq->arm_db); - if (cq->arm_db_index < 0) - goto err_out_ci; - } - } - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - goto err_out_arm; - - cq_context = mailbox->buf; - - if (cq->is_kernel) { - err = mthca_alloc_cq_buf(dev, &cq->buf, nent); - if (err) - goto err_out_mailbox; - } - - spin_lock_init(&cq->lock); - cq->refcount = 1; - init_waitqueue_head(&cq->wait); - mutex_init(&cq->mutex); - - memset(cq_context, 0, sizeof *cq_context); - cq_context->flags = cpu_to_be32(MTHCA_CQ_STATUS_OK | - MTHCA_CQ_STATE_DISARMED | - MTHCA_CQ_FLAG_TR); - cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24); - if (ctx) - cq_context->logsize_usrpage |= cpu_to_be32(ctx->uar.index); - else - cq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index); - cq_context->error_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn); - cq_context->comp_eqn = cpu_to_be32(dev->eq_table.eq[MTHCA_EQ_COMP].eqn); - cq_context->pd = cpu_to_be32(pdn); - cq_context->lkey = cpu_to_be32(cq->buf.mr.ibmr.lkey); - cq_context->cqn = cpu_to_be32(cq->cqn); - - if (mthca_is_memfree(dev)) { - cq_context->ci_db = cpu_to_be32(cq->set_ci_db_index); - cq_context->state_db = cpu_to_be32(cq->arm_db_index); - } - - err = mthca_SW2HW_CQ(dev, mailbox, cq->cqn, &status); - if (err) { - mthca_warn(dev, "SW2HW_CQ failed (%d)\n", err); - goto err_out_free_mr; - } - - if (status) { - mthca_warn(dev, "SW2HW_CQ returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_free_mr; - } - - spin_lock_irq(&dev->cq_table.lock); - if (mthca_array_set(&dev->cq_table.cq, - cq->cqn & (dev->limits.num_cqs - 1), - cq)) { - spin_unlock_irq(&dev->cq_table.lock); - goto err_out_free_mr; - } - spin_unlock_irq(&dev->cq_table.lock); - - cq->cons_index = 0; - - mthca_free_mailbox(dev, mailbox); - - return 0; - -err_out_free_mr: - if (cq->is_kernel) - mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); - -err_out_mailbox: - mthca_free_mailbox(dev, mailbox); - -err_out_arm: - if (cq->is_kernel && mthca_is_memfree(dev)) - mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); - -err_out_ci: - if (cq->is_kernel && mthca_is_memfree(dev)) - mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); - -err_out_icm: - mthca_table_put(dev, dev->cq_table.table, cq->cqn); - -err_out: - mthca_free(&dev->cq_table.alloc, cq->cqn); - - return err; -} - -static inline int get_cq_refcount(struct mthca_dev *dev, struct mthca_cq *cq) -{ - int c; - - spin_lock_irq(&dev->cq_table.lock); - c = cq->refcount; - spin_unlock_irq(&dev->cq_table.lock); - - return c; -} - -void mthca_free_cq(struct mthca_dev *dev, - struct mthca_cq *cq) -{ - struct mthca_mailbox *mailbox; - int err; - u8 status; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - mthca_warn(dev, "No memory for mailbox to free CQ.\n"); - return; - } - - err = mthca_HW2SW_CQ(dev, mailbox, cq->cqn, &status); - if (err) - mthca_warn(dev, "HW2SW_CQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n", status); - - if (0) { - __be32 *ctx = mailbox->buf; - int j; - - printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n", - cq->cqn, cq->cons_index, - cq->is_kernel ? !!next_cqe_sw(cq) : 0); - for (j = 0; j < 16; ++j) - printk(KERN_ERR "[%2x] %08x\n", j * 4, be32_to_cpu(ctx[j])); - } - - spin_lock_irq(&dev->cq_table.lock); - mthca_array_clear(&dev->cq_table.cq, - cq->cqn & (dev->limits.num_cqs - 1)); - --cq->refcount; - spin_unlock_irq(&dev->cq_table.lock); - - if (dev->mthca_flags & MTHCA_FLAG_MSI_X) - synchronize_irq(dev->eq_table.eq[MTHCA_EQ_COMP].msi_x_vector); - else - synchronize_irq(dev->pdev->irq); - - wait_event(cq->wait, !get_cq_refcount(dev, cq)); - - if (cq->is_kernel) { - mthca_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); - if (mthca_is_memfree(dev)) { - mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM, cq->arm_db_index); - mthca_free_db(dev, MTHCA_DB_TYPE_CQ_SET_CI, cq->set_ci_db_index); - } - } - - mthca_table_put(dev, dev->cq_table.table, cq->cqn); - mthca_free(&dev->cq_table.alloc, cq->cqn); - mthca_free_mailbox(dev, mailbox); -} - -int mthca_init_cq_table(struct mthca_dev *dev) -{ - int err; - - spin_lock_init(&dev->cq_table.lock); - - err = mthca_alloc_init(&dev->cq_table.alloc, - dev->limits.num_cqs, - (1 << 24) - 1, - dev->limits.reserved_cqs); - if (err) - return err; - - err = mthca_array_init(&dev->cq_table.cq, - dev->limits.num_cqs); - if (err) - mthca_alloc_cleanup(&dev->cq_table.alloc); - - return err; -} - -void mthca_cleanup_cq_table(struct mthca_dev *dev) -{ - mthca_array_cleanup(&dev->cq_table.cq, dev->limits.num_cqs); - mthca_alloc_cleanup(&dev->cq_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h deleted file mode 100644 index a6dfc6d307d2..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_dev.h +++ /dev/null @@ -1,597 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_DEV_H -#define MTHCA_DEV_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "mthca_provider.h" -#include "mthca_doorbell.h" - -#define DRV_NAME "ib_mthca" -#define PFX DRV_NAME ": " -#define DRV_VERSION "1.0-ofed1.5.2" -#define DRV_RELDATE "August 4, 2010" - -enum { - MTHCA_FLAG_DDR_HIDDEN = 1 << 1, - MTHCA_FLAG_SRQ = 1 << 2, - MTHCA_FLAG_MSI_X = 1 << 3, - MTHCA_FLAG_NO_LAM = 1 << 4, - MTHCA_FLAG_FMR = 1 << 5, - MTHCA_FLAG_MEMFREE = 1 << 6, - MTHCA_FLAG_PCIE = 1 << 7, - MTHCA_FLAG_SINAI_OPT = 1 << 8 -}; - -enum { - MTHCA_MAX_PORTS = 2 -}; - -enum { - MTHCA_BOARD_ID_LEN = 64 -}; - -enum { - MTHCA_EQ_CONTEXT_SIZE = 0x40, - MTHCA_CQ_CONTEXT_SIZE = 0x40, - MTHCA_QP_CONTEXT_SIZE = 0x200, - MTHCA_RDB_ENTRY_SIZE = 0x20, - MTHCA_AV_SIZE = 0x20, - MTHCA_MGM_ENTRY_SIZE = 0x100, - - /* Arbel FW gives us these, but we need them for Tavor */ - MTHCA_MPT_ENTRY_SIZE = 0x40, - MTHCA_MTT_SEG_SIZE = 0x40, - - MTHCA_QP_PER_MGM = 4 * (MTHCA_MGM_ENTRY_SIZE / 16 - 2) -}; - -enum { - MTHCA_EQ_CMD, - MTHCA_EQ_ASYNC, - MTHCA_EQ_COMP, - MTHCA_NUM_EQ -}; - -enum { - MTHCA_OPCODE_NOP = 0x00, - MTHCA_OPCODE_RDMA_WRITE = 0x08, - MTHCA_OPCODE_RDMA_WRITE_IMM = 0x09, - MTHCA_OPCODE_SEND = 0x0a, - MTHCA_OPCODE_SEND_IMM = 0x0b, - MTHCA_OPCODE_RDMA_READ = 0x10, - MTHCA_OPCODE_ATOMIC_CS = 0x11, - MTHCA_OPCODE_ATOMIC_FA = 0x12, - MTHCA_OPCODE_BIND_MW = 0x18, - MTHCA_OPCODE_INVALID = 0xff -}; - -enum { - MTHCA_CMD_USE_EVENTS = 1 << 0, - MTHCA_CMD_POST_DOORBELLS = 1 << 1 -}; - -enum { - MTHCA_CMD_NUM_DBELL_DWORDS = 8 -}; - -struct mthca_cmd { - struct pci_pool *pool; - struct mutex hcr_mutex; - struct semaphore poll_sem; - struct semaphore event_sem; - int max_cmds; - spinlock_t context_lock; - int free_head; - struct mthca_cmd_context *context; - u16 token_mask; - u32 flags; - void __iomem *dbell_map; - u16 dbell_offsets[MTHCA_CMD_NUM_DBELL_DWORDS]; -}; - -struct mthca_limits { - int num_ports; - int vl_cap; - int mtu_cap; - int gid_table_len; - int pkey_table_len; - int local_ca_ack_delay; - int num_uars; - int max_sg; - int num_qps; - int max_wqes; - int max_desc_sz; - int max_qp_init_rdma; - int reserved_qps; - int num_srqs; - int max_srq_wqes; - int max_srq_sge; - int reserved_srqs; - int num_eecs; - int reserved_eecs; - int num_cqs; - int max_cqes; - int reserved_cqs; - int num_eqs; - int reserved_eqs; - int num_mpts; - int num_mtt_segs; - int mtt_seg_size; - int fmr_reserved_mtts; - int reserved_mtts; - int reserved_mrws; - int reserved_uars; - int num_mgms; - int num_amgms; - int reserved_mcgs; - int num_pds; - int reserved_pds; - u32 page_size_cap; - u32 flags; - u16 stat_rate_support; - u8 port_width_cap; -}; - -struct mthca_alloc { - u32 last; - u32 top; - u32 max; - u32 mask; - spinlock_t lock; - unsigned long *table; -}; - -struct mthca_array { - struct { - void **page; - int used; - } *page_list; -}; - -struct mthca_uar_table { - struct mthca_alloc alloc; - u64 uarc_base; - int uarc_size; -}; - -struct mthca_pd_table { - struct mthca_alloc alloc; -}; - -struct mthca_buddy { - unsigned long **bits; - int *num_free; - int max_order; - spinlock_t lock; -}; - -struct mthca_mr_table { - struct mthca_alloc mpt_alloc; - struct mthca_buddy mtt_buddy; - struct mthca_buddy *fmr_mtt_buddy; - u64 mtt_base; - u64 mpt_base; - struct mthca_icm_table *mtt_table; - struct mthca_icm_table *mpt_table; - struct { - void __iomem *mpt_base; - void __iomem *mtt_base; - struct mthca_buddy mtt_buddy; - } tavor_fmr; -}; - -struct mthca_eq_table { - struct mthca_alloc alloc; - void __iomem *clr_int; - u32 clr_mask; - u32 arm_mask; - struct mthca_eq eq[MTHCA_NUM_EQ]; - u64 icm_virt; - struct page *icm_page; - dma_addr_t icm_dma; - int have_irq; - u8 inta_pin; -}; - -struct mthca_cq_table { - struct mthca_alloc alloc; - spinlock_t lock; - struct mthca_array cq; - struct mthca_icm_table *table; -}; - -struct mthca_srq_table { - struct mthca_alloc alloc; - spinlock_t lock; - struct mthca_array srq; - struct mthca_icm_table *table; -}; - -struct mthca_qp_table { - struct mthca_alloc alloc; - u32 rdb_base; - int rdb_shift; - int sqp_start; - spinlock_t lock; - struct mthca_array qp; - struct mthca_icm_table *qp_table; - struct mthca_icm_table *eqp_table; - struct mthca_icm_table *rdb_table; -}; - -struct mthca_av_table { - struct pci_pool *pool; - int num_ddr_avs; - u64 ddr_av_base; - void __iomem *av_map; - struct mthca_alloc alloc; -}; - -struct mthca_mcg_table { - struct mutex mutex; - struct mthca_alloc alloc; - struct mthca_icm_table *table; -}; - -struct mthca_catas_err { - u64 addr; - u32 __iomem *map; - u32 size; - struct timer_list timer; - struct list_head list; -}; - -extern struct mutex mthca_device_mutex; - -struct mthca_dev { - struct ib_device ib_dev; - struct pci_dev *pdev; - - int hca_type; - unsigned long mthca_flags; - unsigned long device_cap_flags; - - u32 rev_id; - char board_id[MTHCA_BOARD_ID_LEN]; - - /* firmware info */ - u64 fw_ver; - union { - struct { - u64 fw_start; - u64 fw_end; - } tavor; - struct { - u64 clr_int_base; - u64 eq_arm_base; - u64 eq_set_ci_base; - struct mthca_icm *fw_icm; - struct mthca_icm *aux_icm; - u16 fw_pages; - } arbel; - } fw; - - u64 ddr_start; - u64 ddr_end; - - MTHCA_DECLARE_DOORBELL_LOCK(doorbell_lock) - struct mutex cap_mask_mutex; - - void __iomem *hcr; - void __iomem *kar; - void __iomem *clr_base; - union { - struct { - void __iomem *ecr_base; - } tavor; - struct { - void __iomem *eq_arm; - void __iomem *eq_set_ci_base; - } arbel; - } eq_regs; - - struct mthca_cmd cmd; - struct mthca_limits limits; - - struct mthca_uar_table uar_table; - struct mthca_pd_table pd_table; - struct mthca_mr_table mr_table; - struct mthca_eq_table eq_table; - struct mthca_cq_table cq_table; - struct mthca_srq_table srq_table; - struct mthca_qp_table qp_table; - struct mthca_av_table av_table; - struct mthca_mcg_table mcg_table; - - struct mthca_catas_err catas_err; - - struct mthca_uar driver_uar; - struct mthca_db_table *db_tab; - struct mthca_pd driver_pd; - struct mthca_mr driver_mr; - - struct ib_mad_agent *send_agent[MTHCA_MAX_PORTS][2]; - struct ib_ah *sm_ah[MTHCA_MAX_PORTS]; - spinlock_t sm_lock; - u8 rate[MTHCA_MAX_PORTS]; - int active; -}; - -#ifdef CONFIG_INFINIBAND_MTHCA_DEBUG -extern int mthca_debug_level; - -#define mthca_dbg(mdev, format, arg...) \ - do { \ - if (mthca_debug_level) \ - dev_printk(KERN_DEBUG, &mdev->pdev->dev, format, ## arg); \ - } while (0) - -#else /* CONFIG_INFINIBAND_MTHCA_DEBUG */ - -#define mthca_dbg(mdev, format, arg...) do { (void) mdev; } while (0) - -#endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */ - -#define mthca_err(mdev, format, arg...) \ - dev_err(&mdev->pdev->dev, format, ## arg) -#define mthca_info(mdev, format, arg...) \ - dev_info(&mdev->pdev->dev, format, ## arg) -#define mthca_warn(mdev, format, arg...) \ - dev_warn(&mdev->pdev->dev, format, ## arg) - -extern void __buggy_use_of_MTHCA_GET(void); -extern void __buggy_use_of_MTHCA_PUT(void); - -#define MTHCA_GET(dest, source, offset) \ - do { \ - void *__p = (char *) (source) + (offset); \ - switch (sizeof (dest)) { \ - case 1: (dest) = *(u8 *) __p; break; \ - case 2: (dest) = be16_to_cpup(__p); break; \ - case 4: (dest) = be32_to_cpup(__p); break; \ - case 8: (dest) = be64_to_cpup(__p); break; \ - default: __buggy_use_of_MTHCA_GET(); \ - } \ - } while (0) - -#define MTHCA_PUT(dest, source, offset) \ - do { \ - void *__d = ((char *) (dest) + (offset)); \ - switch (sizeof(source)) { \ - case 1: *(u8 *) __d = (source); break; \ - case 2: *(__be16 *) __d = cpu_to_be16(source); break; \ - case 4: *(__be32 *) __d = cpu_to_be32(source); break; \ - case 8: *(__be64 *) __d = cpu_to_be64(source); break; \ - default: __buggy_use_of_MTHCA_PUT(); \ - } \ - } while (0) - -int mthca_reset(struct mthca_dev *mdev); - -u32 mthca_alloc(struct mthca_alloc *alloc); -void mthca_free(struct mthca_alloc *alloc, u32 obj); -int mthca_alloc_init(struct mthca_alloc *alloc, u32 num, u32 mask, - u32 reserved); -void mthca_alloc_cleanup(struct mthca_alloc *alloc); -void *mthca_array_get(struct mthca_array *array, int index); -int mthca_array_set(struct mthca_array *array, int index, void *value); -void mthca_array_clear(struct mthca_array *array, int index); -int mthca_array_init(struct mthca_array *array, int nent); -void mthca_array_cleanup(struct mthca_array *array, int nent); -int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, - union mthca_buf *buf, int *is_direct, struct mthca_pd *pd, - int hca_write, struct mthca_mr *mr); -void mthca_buf_free(struct mthca_dev *dev, int size, union mthca_buf *buf, - int is_direct, struct mthca_mr *mr); - -int mthca_init_uar_table(struct mthca_dev *dev); -int mthca_init_pd_table(struct mthca_dev *dev); -int mthca_init_mr_table(struct mthca_dev *dev); -int mthca_init_eq_table(struct mthca_dev *dev); -int mthca_init_cq_table(struct mthca_dev *dev); -int mthca_init_srq_table(struct mthca_dev *dev); -int mthca_init_qp_table(struct mthca_dev *dev); -int mthca_init_av_table(struct mthca_dev *dev); -int mthca_init_mcg_table(struct mthca_dev *dev); - -void mthca_cleanup_uar_table(struct mthca_dev *dev); -void mthca_cleanup_pd_table(struct mthca_dev *dev); -void mthca_cleanup_mr_table(struct mthca_dev *dev); -void mthca_cleanup_eq_table(struct mthca_dev *dev); -void mthca_cleanup_cq_table(struct mthca_dev *dev); -void mthca_cleanup_srq_table(struct mthca_dev *dev); -void mthca_cleanup_qp_table(struct mthca_dev *dev); -void mthca_cleanup_av_table(struct mthca_dev *dev); -void mthca_cleanup_mcg_table(struct mthca_dev *dev); - -int mthca_register_device(struct mthca_dev *dev); -void mthca_unregister_device(struct mthca_dev *dev); - -void mthca_start_catas_poll(struct mthca_dev *dev); -void mthca_stop_catas_poll(struct mthca_dev *dev); -int __mthca_restart_one(struct pci_dev *pdev); -int mthca_catas_init(void); -void mthca_catas_cleanup(void); - -int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar); -void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar); - -int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd); -void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd); - -int mthca_write_mtt_size(struct mthca_dev *dev); - -struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size); -void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt); -int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, - int start_index, u64 *buffer_list, int list_len); -int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, - u64 iova, u64 total_size, u32 access, struct mthca_mr *mr); -int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, - u32 access, struct mthca_mr *mr); -int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, - u64 *buffer_list, int buffer_size_shift, - int list_len, u64 iova, u64 total_size, - u32 access, struct mthca_mr *mr); -void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr); - -int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, - u32 access, struct mthca_fmr *fmr); -int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, - int list_len, u64 iova); -void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr); -int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, - int list_len, u64 iova); -void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr); -int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr); - -int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt); -void mthca_unmap_eq_icm(struct mthca_dev *dev); - -int mthca_poll_cq(struct ib_cq *ibcq, int num_entries, - struct ib_wc *entry); -int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); -int mthca_arbel_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); -int mthca_init_cq(struct mthca_dev *dev, int nent, - struct mthca_ucontext *ctx, u32 pdn, - struct mthca_cq *cq); -void mthca_free_cq(struct mthca_dev *dev, - struct mthca_cq *cq); -void mthca_cq_completion(struct mthca_dev *dev, u32 cqn); -void mthca_cq_event(struct mthca_dev *dev, u32 cqn, - enum ib_event_type event_type); -void mthca_cq_clean(struct mthca_dev *dev, struct mthca_cq *cq, u32 qpn, - struct mthca_srq *srq); -void mthca_cq_resize_copy_cqes(struct mthca_cq *cq); -int mthca_alloc_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int nent); -void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq_buf *buf, int cqe); - -int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, - struct ib_srq_attr *attr, struct mthca_srq *srq); -void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq); -int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, - enum ib_srq_attr_mask attr_mask, struct ib_udata *udata); -int mthca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr); -int mthca_max_srq_sge(struct mthca_dev *dev); -void mthca_srq_event(struct mthca_dev *dev, u32 srqn, - enum ib_event_type event_type); -void mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr); -int mthca_tavor_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); -int mthca_arbel_post_srq_recv(struct ib_srq *srq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); - -void mthca_qp_event(struct mthca_dev *dev, u32 qpn, - enum ib_event_type event_type); -int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr); -int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, - struct ib_udata *udata); -int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr); -int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); -int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr); -int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr); -void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, - int index, int *dbd, __be32 *new_wqe); -int mthca_alloc_qp(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_cq *send_cq, - struct mthca_cq *recv_cq, - enum ib_qp_type type, - enum ib_sig_type send_policy, - struct ib_qp_cap *cap, - struct mthca_qp *qp); -int mthca_alloc_sqp(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_cq *send_cq, - struct mthca_cq *recv_cq, - enum ib_sig_type send_policy, - struct ib_qp_cap *cap, - int qpn, - int port, - struct mthca_sqp *sqp); -void mthca_free_qp(struct mthca_dev *dev, struct mthca_qp *qp); -int mthca_create_ah(struct mthca_dev *dev, - struct mthca_pd *pd, - struct ib_ah_attr *ah_attr, - struct mthca_ah *ah); -int mthca_destroy_ah(struct mthca_dev *dev, struct mthca_ah *ah); -int mthca_read_ah(struct mthca_dev *dev, struct mthca_ah *ah, - struct ib_ud_header *header); -int mthca_ah_query(struct ib_ah *ibah, struct ib_ah_attr *attr); -int mthca_ah_grh_present(struct mthca_ah *ah); -u8 mthca_get_rate(struct mthca_dev *dev, int static_rate, u8 port); -enum ib_rate mthca_rate_to_ib(struct mthca_dev *dev, u8 mthca_rate, u8 port); - -int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid); -int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid); - -int mthca_process_mad(struct ib_device *ibdev, - int mad_flags, - u8 port_num, - struct ib_wc *in_wc, - struct ib_grh *in_grh, - struct ib_mad *in_mad, - struct ib_mad *out_mad); -int mthca_create_agents(struct mthca_dev *dev); -void mthca_free_agents(struct mthca_dev *dev); - -static inline struct mthca_dev *to_mdev(struct ib_device *ibdev) -{ - return container_of(ibdev, struct mthca_dev, ib_dev); -} - -static inline int mthca_is_memfree(struct mthca_dev *dev) -{ - return dev->mthca_flags & MTHCA_FLAG_MEMFREE; -} - -#endif /* MTHCA_DEV_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_doorbell.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_doorbell.h deleted file mode 100644 index 14f51ef97d7e..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_doorbell.h +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#define MTHCA_RD_DOORBELL 0x00 -#define MTHCA_SEND_DOORBELL 0x10 -#define MTHCA_RECEIVE_DOORBELL 0x18 -#define MTHCA_CQ_DOORBELL 0x20 -#define MTHCA_EQ_DOORBELL 0x28 - -#if BITS_PER_LONG == 64 -/* - * Assume that we can just write a 64-bit doorbell atomically. s390 - * actually doesn't have writeq() but S/390 systems don't even have - * PCI so we won't worry about it. - */ - -#define MTHCA_DECLARE_DOORBELL_LOCK(name) -#define MTHCA_INIT_DOORBELL_LOCK(ptr) do { } while (0) -#define MTHCA_GET_DOORBELL_LOCK(ptr) (NULL) - -static inline void mthca_write64_raw(__be64 val, void __iomem *dest) -{ - __raw_writeq((__force u64) val, dest); -} - -static inline void mthca_write64(u32 hi, u32 lo, void __iomem *dest, - spinlock_t *doorbell_lock) -{ - __raw_writeq((__force u64) cpu_to_be64((u64) hi << 32 | lo), dest); -} - -static inline void mthca_write_db_rec(__be32 val[2], __be32 *db) -{ - *(u64 *) db = *(u64 *) val; -} - -#else - -/* - * Just fall back to a spinlock to protect the doorbell if - * BITS_PER_LONG is 32 -- there's no portable way to do atomic 64-bit - * MMIO writes. - */ - -#define MTHCA_DECLARE_DOORBELL_LOCK(name) spinlock_t name; -#define MTHCA_INIT_DOORBELL_LOCK(ptr) spin_lock_init(ptr) -#define MTHCA_GET_DOORBELL_LOCK(ptr) (ptr) - -static inline void mthca_write64_raw(__be64 val, void __iomem *dest) -{ - __raw_writel(((__force u32 *) &val)[0], dest); - __raw_writel(((__force u32 *) &val)[1], dest + 4); -} - -static inline void mthca_write64(u32 hi, u32 lo, void __iomem *dest, - spinlock_t *doorbell_lock) -{ - unsigned long flags; - - hi = (__force u32) cpu_to_be32(hi); - lo = (__force u32) cpu_to_be32(lo); - - spin_lock_irqsave(doorbell_lock, flags); - __raw_writel(hi, dest); - __raw_writel(lo, dest + 4); - spin_unlock_irqrestore(doorbell_lock, flags); -} - -static inline void mthca_write_db_rec(__be32 val[2], __be32 *db) -{ - db[0] = val[0]; - wmb(); - db[1] = val[1]; -} - -#endif diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c deleted file mode 100644 index 90e4e450a120..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_eq.c +++ /dev/null @@ -1,920 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_config_reg.h" - -enum { - MTHCA_NUM_ASYNC_EQE = 0x80, - MTHCA_NUM_CMD_EQE = 0x80, - MTHCA_NUM_SPARE_EQE = 0x80, - MTHCA_EQ_ENTRY_SIZE = 0x20 -}; - -/* - * Must be packed because start is 64 bits but only aligned to 32 bits. - */ -struct mthca_eq_context { - __be32 flags; - __be64 start; - __be32 logsize_usrpage; - __be32 tavor_pd; /* reserved for Arbel */ - u8 reserved1[3]; - u8 intr; - __be32 arbel_pd; /* lost_count for Tavor */ - __be32 lkey; - u32 reserved2[2]; - __be32 consumer_index; - __be32 producer_index; - u32 reserved3[4]; -} __attribute__((packed)); - -#define MTHCA_EQ_STATUS_OK ( 0 << 28) -#define MTHCA_EQ_STATUS_OVERFLOW ( 9 << 28) -#define MTHCA_EQ_STATUS_WRITE_FAIL (10 << 28) -#define MTHCA_EQ_OWNER_SW ( 0 << 24) -#define MTHCA_EQ_OWNER_HW ( 1 << 24) -#define MTHCA_EQ_FLAG_TR ( 1 << 18) -#define MTHCA_EQ_FLAG_OI ( 1 << 17) -#define MTHCA_EQ_STATE_ARMED ( 1 << 8) -#define MTHCA_EQ_STATE_FIRED ( 2 << 8) -#define MTHCA_EQ_STATE_ALWAYS_ARMED ( 3 << 8) -#define MTHCA_EQ_STATE_ARBEL ( 8 << 8) - -enum { - MTHCA_EVENT_TYPE_COMP = 0x00, - MTHCA_EVENT_TYPE_PATH_MIG = 0x01, - MTHCA_EVENT_TYPE_COMM_EST = 0x02, - MTHCA_EVENT_TYPE_SQ_DRAINED = 0x03, - MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE = 0x13, - MTHCA_EVENT_TYPE_SRQ_LIMIT = 0x14, - MTHCA_EVENT_TYPE_CQ_ERROR = 0x04, - MTHCA_EVENT_TYPE_WQ_CATAS_ERROR = 0x05, - MTHCA_EVENT_TYPE_EEC_CATAS_ERROR = 0x06, - MTHCA_EVENT_TYPE_PATH_MIG_FAILED = 0x07, - MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR = 0x10, - MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR = 0x11, - MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR = 0x12, - MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR = 0x08, - MTHCA_EVENT_TYPE_PORT_CHANGE = 0x09, - MTHCA_EVENT_TYPE_EQ_OVERFLOW = 0x0f, - MTHCA_EVENT_TYPE_ECC_DETECT = 0x0e, - MTHCA_EVENT_TYPE_CMD = 0x0a -}; - -#define MTHCA_ASYNC_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_PATH_MIG) | \ - (1ULL << MTHCA_EVENT_TYPE_COMM_EST) | \ - (1ULL << MTHCA_EVENT_TYPE_SQ_DRAINED) | \ - (1ULL << MTHCA_EVENT_TYPE_CQ_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_WQ_CATAS_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_EEC_CATAS_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_PATH_MIG_FAILED) | \ - (1ULL << MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_PORT_CHANGE) | \ - (1ULL << MTHCA_EVENT_TYPE_ECC_DETECT)) -#define MTHCA_SRQ_EVENT_MASK ((1ULL << MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR) | \ - (1ULL << MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE) | \ - (1ULL << MTHCA_EVENT_TYPE_SRQ_LIMIT)) -#define MTHCA_CMD_EVENT_MASK (1ULL << MTHCA_EVENT_TYPE_CMD) - -#define MTHCA_EQ_DB_INC_CI (1 << 24) -#define MTHCA_EQ_DB_REQ_NOT (2 << 24) -#define MTHCA_EQ_DB_DISARM_CQ (3 << 24) -#define MTHCA_EQ_DB_SET_CI (4 << 24) -#define MTHCA_EQ_DB_ALWAYS_ARM (5 << 24) - -struct mthca_eqe { - u8 reserved1; - u8 type; - u8 reserved2; - u8 subtype; - union { - u32 raw[6]; - struct { - __be32 cqn; - } __attribute__((packed)) comp; - struct { - u16 reserved1; - __be16 token; - u32 reserved2; - u8 reserved3[3]; - u8 status; - __be64 out_param; - } __attribute__((packed)) cmd; - struct { - __be32 qpn; - } __attribute__((packed)) qp; - struct { - __be32 srqn; - } __attribute__((packed)) srq; - struct { - __be32 cqn; - u32 reserved1; - u8 reserved2[3]; - u8 syndrome; - } __attribute__((packed)) cq_err; - struct { - u32 reserved1[2]; - __be32 port; - } __attribute__((packed)) port_change; - } event; - u8 reserved3[3]; - u8 owner; -} __attribute__((packed)); - -#define MTHCA_EQ_ENTRY_OWNER_SW (0 << 7) -#define MTHCA_EQ_ENTRY_OWNER_HW (1 << 7) - -static inline u64 async_mask(struct mthca_dev *dev) -{ - return dev->mthca_flags & MTHCA_FLAG_SRQ ? - MTHCA_ASYNC_EVENT_MASK | MTHCA_SRQ_EVENT_MASK : - MTHCA_ASYNC_EVENT_MASK; -} - -static inline void tavor_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci) -{ - /* - * This barrier makes sure that all updates to ownership bits - * done by set_eqe_hw() hit memory before the consumer index - * is updated. set_eq_ci() allows the HCA to possibly write - * more EQ entries, and we want to avoid the exceedingly - * unlikely possibility of the HCA writing an entry and then - * having set_eqe_hw() overwrite the owner field. - */ - wmb(); - mthca_write64(MTHCA_EQ_DB_SET_CI | eq->eqn, ci & (eq->nent - 1), - dev->kar + MTHCA_EQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); -} - -static inline void arbel_set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci) -{ - /* See comment in tavor_set_eq_ci() above. */ - wmb(); - __raw_writel((__force u32) cpu_to_be32(ci), - dev->eq_regs.arbel.eq_set_ci_base + eq->eqn * 8); - /* We still want ordering, just not swabbing, so add a barrier */ - mb(); -} - -static inline void set_eq_ci(struct mthca_dev *dev, struct mthca_eq *eq, u32 ci) -{ - if (mthca_is_memfree(dev)) - arbel_set_eq_ci(dev, eq, ci); - else - tavor_set_eq_ci(dev, eq, ci); -} - -static inline void tavor_eq_req_not(struct mthca_dev *dev, int eqn) -{ - mthca_write64(MTHCA_EQ_DB_REQ_NOT | eqn, 0, - dev->kar + MTHCA_EQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); -} - -static inline void arbel_eq_req_not(struct mthca_dev *dev, u32 eqn_mask) -{ - writel(eqn_mask, dev->eq_regs.arbel.eq_arm); -} - -static inline void disarm_cq(struct mthca_dev *dev, int eqn, int cqn) -{ - if (!mthca_is_memfree(dev)) { - mthca_write64(MTHCA_EQ_DB_DISARM_CQ | eqn, cqn, - dev->kar + MTHCA_EQ_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - } -} - -static inline struct mthca_eqe *get_eqe(struct mthca_eq *eq, u32 entry) -{ - unsigned long off = (entry & (eq->nent - 1)) * MTHCA_EQ_ENTRY_SIZE; - return eq->page_list[off / PAGE_SIZE].buf + off % PAGE_SIZE; -} - -static inline struct mthca_eqe *next_eqe_sw(struct mthca_eq *eq) -{ - struct mthca_eqe *eqe; - eqe = get_eqe(eq, eq->cons_index); - return (MTHCA_EQ_ENTRY_OWNER_HW & eqe->owner) ? NULL : eqe; -} - -static inline void set_eqe_hw(struct mthca_eqe *eqe) -{ - eqe->owner = MTHCA_EQ_ENTRY_OWNER_HW; -} - -static void port_change(struct mthca_dev *dev, int port, int active) -{ - struct ib_event record; - - mthca_dbg(dev, "Port change to %s for port %d\n", - active ? "active" : "down", port); - - record.device = &dev->ib_dev; - record.event = active ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; - record.element.port_num = port; - - ib_dispatch_event(&record); -} - -static int mthca_eq_int(struct mthca_dev *dev, struct mthca_eq *eq) -{ - struct mthca_eqe *eqe; - int disarm_cqn; - int eqes_found = 0; - int set_ci = 0; - - while ((eqe = next_eqe_sw(eq))) { - /* - * Make sure we read EQ entry contents after we've - * checked the ownership bit. - */ - rmb(); - - switch (eqe->type) { - case MTHCA_EVENT_TYPE_COMP: - disarm_cqn = be32_to_cpu(eqe->event.comp.cqn) & 0xffffff; - disarm_cq(dev, eq->eqn, disarm_cqn); - mthca_cq_completion(dev, disarm_cqn); - break; - - case MTHCA_EVENT_TYPE_PATH_MIG: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_PATH_MIG); - break; - - case MTHCA_EVENT_TYPE_COMM_EST: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_COMM_EST); - break; - - case MTHCA_EVENT_TYPE_SQ_DRAINED: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_SQ_DRAINED); - break; - - case MTHCA_EVENT_TYPE_SRQ_QP_LAST_WQE: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_QP_LAST_WQE_REACHED); - break; - - case MTHCA_EVENT_TYPE_SRQ_LIMIT: - mthca_srq_event(dev, be32_to_cpu(eqe->event.srq.srqn) & 0xffffff, - IB_EVENT_SRQ_LIMIT_REACHED); - break; - - case MTHCA_EVENT_TYPE_WQ_CATAS_ERROR: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_QP_FATAL); - break; - - case MTHCA_EVENT_TYPE_PATH_MIG_FAILED: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_PATH_MIG_ERR); - break; - - case MTHCA_EVENT_TYPE_WQ_INVAL_REQ_ERROR: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_QP_REQ_ERR); - break; - - case MTHCA_EVENT_TYPE_WQ_ACCESS_ERROR: - mthca_qp_event(dev, be32_to_cpu(eqe->event.qp.qpn) & 0xffffff, - IB_EVENT_QP_ACCESS_ERR); - break; - - case MTHCA_EVENT_TYPE_CMD: - mthca_cmd_event(dev, - be16_to_cpu(eqe->event.cmd.token), - eqe->event.cmd.status, - be64_to_cpu(eqe->event.cmd.out_param)); - break; - - case MTHCA_EVENT_TYPE_PORT_CHANGE: - port_change(dev, - (be32_to_cpu(eqe->event.port_change.port) >> 28) & 3, - eqe->subtype == 0x4); - break; - - case MTHCA_EVENT_TYPE_CQ_ERROR: - mthca_warn(dev, "CQ %s on CQN %06x\n", - eqe->event.cq_err.syndrome == 1 ? - "overrun" : "access violation", - be32_to_cpu(eqe->event.cq_err.cqn) & 0xffffff); - mthca_cq_event(dev, be32_to_cpu(eqe->event.cq_err.cqn), - IB_EVENT_CQ_ERR); - break; - - case MTHCA_EVENT_TYPE_EQ_OVERFLOW: - mthca_warn(dev, "EQ overrun on EQN %d\n", eq->eqn); - break; - - case MTHCA_EVENT_TYPE_EEC_CATAS_ERROR: - case MTHCA_EVENT_TYPE_SRQ_CATAS_ERROR: - case MTHCA_EVENT_TYPE_LOCAL_CATAS_ERROR: - case MTHCA_EVENT_TYPE_ECC_DETECT: - default: - mthca_warn(dev, "Unhandled event %02x(%02x) on EQ %d\n", - eqe->type, eqe->subtype, eq->eqn); - break; - }; - - set_eqe_hw(eqe); - ++eq->cons_index; - eqes_found = 1; - ++set_ci; - - /* - * The HCA will think the queue has overflowed if we - * don't tell it we've been processing events. We - * create our EQs with MTHCA_NUM_SPARE_EQE extra - * entries, so we must update our consumer index at - * least that often. - */ - if (unlikely(set_ci >= MTHCA_NUM_SPARE_EQE)) { - /* - * Conditional on hca_type is OK here because - * this is a rare case, not the fast path. - */ - set_eq_ci(dev, eq, eq->cons_index); - set_ci = 0; - } - } - - /* - * Rely on caller to set consumer index so that we don't have - * to test hca_type in our interrupt handling fast path. - */ - return eqes_found; -} - -static irqreturn_t mthca_tavor_interrupt(int irq, void *dev_ptr) -{ - struct mthca_dev *dev = dev_ptr; - u32 ecr; - int i; - - if (dev->eq_table.clr_mask) - writel(dev->eq_table.clr_mask, dev->eq_table.clr_int); - - ecr = readl(dev->eq_regs.tavor.ecr_base + 4); - if (!ecr) - return IRQ_NONE; - - writel(ecr, dev->eq_regs.tavor.ecr_base + - MTHCA_ECR_CLR_BASE - MTHCA_ECR_BASE + 4); - - for (i = 0; i < MTHCA_NUM_EQ; ++i) - if (ecr & dev->eq_table.eq[i].eqn_mask) { - if (mthca_eq_int(dev, &dev->eq_table.eq[i])) - tavor_set_eq_ci(dev, &dev->eq_table.eq[i], - dev->eq_table.eq[i].cons_index); - tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn); - } - - return IRQ_HANDLED; -} - -static irqreturn_t mthca_tavor_msi_x_interrupt(int irq, void *eq_ptr) -{ - struct mthca_eq *eq = eq_ptr; - struct mthca_dev *dev = eq->dev; - - mthca_eq_int(dev, eq); - tavor_set_eq_ci(dev, eq, eq->cons_index); - tavor_eq_req_not(dev, eq->eqn); - - /* MSI-X vectors always belong to us */ - return IRQ_HANDLED; -} - -static irqreturn_t mthca_arbel_interrupt(int irq, void *dev_ptr) -{ - struct mthca_dev *dev = dev_ptr; - int work = 0; - int i; - - if (dev->eq_table.clr_mask) - writel(dev->eq_table.clr_mask, dev->eq_table.clr_int); - - for (i = 0; i < MTHCA_NUM_EQ; ++i) - if (mthca_eq_int(dev, &dev->eq_table.eq[i])) { - work = 1; - arbel_set_eq_ci(dev, &dev->eq_table.eq[i], - dev->eq_table.eq[i].cons_index); - } - - arbel_eq_req_not(dev, dev->eq_table.arm_mask); - - return IRQ_RETVAL(work); -} - -static irqreturn_t mthca_arbel_msi_x_interrupt(int irq, void *eq_ptr) -{ - struct mthca_eq *eq = eq_ptr; - struct mthca_dev *dev = eq->dev; - - mthca_eq_int(dev, eq); - arbel_set_eq_ci(dev, eq, eq->cons_index); - arbel_eq_req_not(dev, eq->eqn_mask); - - /* MSI-X vectors always belong to us */ - return IRQ_HANDLED; -} - -static int mthca_create_eq(struct mthca_dev *dev, - int nent, - u8 intr, - struct mthca_eq *eq) -{ - int npages; - u64 *dma_list = NULL; - dma_addr_t t; - struct mthca_mailbox *mailbox; - struct mthca_eq_context *eq_context; - int err = -ENOMEM; - int i; - u8 status; - - eq->dev = dev; - eq->nent = roundup_pow_of_two(max(nent, 2)); - npages = ALIGN(eq->nent * MTHCA_EQ_ENTRY_SIZE, PAGE_SIZE) / PAGE_SIZE; - - eq->page_list = kmalloc(npages * sizeof *eq->page_list, - GFP_KERNEL); - if (!eq->page_list) - goto err_out; - - for (i = 0; i < npages; ++i) - eq->page_list[i].buf = NULL; - - dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL); - if (!dma_list) - goto err_out_free; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - goto err_out_free; - eq_context = mailbox->buf; - - for (i = 0; i < npages; ++i) { - eq->page_list[i].buf = dma_alloc_coherent(&dev->pdev->dev, - PAGE_SIZE, &t, GFP_KERNEL); - if (!eq->page_list[i].buf) - goto err_out_free_pages; - - dma_list[i] = t; - pci_unmap_addr_set(&eq->page_list[i], mapping, t); - - clear_page(eq->page_list[i].buf); - } - - for (i = 0; i < eq->nent; ++i) - set_eqe_hw(get_eqe(eq, i)); - - eq->eqn = mthca_alloc(&dev->eq_table.alloc); - if (eq->eqn == -1) - goto err_out_free_pages; - - err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num, - dma_list, PAGE_SHIFT, npages, - 0, npages * PAGE_SIZE, - MTHCA_MPT_FLAG_LOCAL_WRITE | - MTHCA_MPT_FLAG_LOCAL_READ, - &eq->mr); - if (err) - goto err_out_free_eq; - - memset(eq_context, 0, sizeof *eq_context); - eq_context->flags = cpu_to_be32(MTHCA_EQ_STATUS_OK | - MTHCA_EQ_OWNER_HW | - MTHCA_EQ_STATE_ARMED | - MTHCA_EQ_FLAG_TR); - if (mthca_is_memfree(dev)) - eq_context->flags |= cpu_to_be32(MTHCA_EQ_STATE_ARBEL); - - eq_context->logsize_usrpage = cpu_to_be32((ffs(eq->nent) - 1) << 24); - if (mthca_is_memfree(dev)) { - eq_context->arbel_pd = cpu_to_be32(dev->driver_pd.pd_num); - } else { - eq_context->logsize_usrpage |= cpu_to_be32(dev->driver_uar.index); - eq_context->tavor_pd = cpu_to_be32(dev->driver_pd.pd_num); - } - eq_context->intr = intr; - eq_context->lkey = cpu_to_be32(eq->mr.ibmr.lkey); - - err = mthca_SW2HW_EQ(dev, mailbox, eq->eqn, &status); - if (err) { - mthca_warn(dev, "SW2HW_EQ failed (%d)\n", err); - goto err_out_free_mr; - } - if (status) { - mthca_warn(dev, "SW2HW_EQ returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_free_mr; - } - - kfree(dma_list); - mthca_free_mailbox(dev, mailbox); - - eq->eqn_mask = swab32(1 << eq->eqn); - eq->cons_index = 0; - - dev->eq_table.arm_mask |= eq->eqn_mask; - - mthca_dbg(dev, "Allocated EQ %d with %d entries\n", - eq->eqn, eq->nent); - - return err; - - err_out_free_mr: - mthca_free_mr(dev, &eq->mr); - - err_out_free_eq: - mthca_free(&dev->eq_table.alloc, eq->eqn); - - err_out_free_pages: - for (i = 0; i < npages; ++i) - if (eq->page_list[i].buf) - dma_free_coherent(&dev->pdev->dev, PAGE_SIZE, - eq->page_list[i].buf, - pci_unmap_addr(&eq->page_list[i], - mapping)); - - mthca_free_mailbox(dev, mailbox); - - err_out_free: - kfree(eq->page_list); - kfree(dma_list); - - err_out: - return err; -} - -static void mthca_free_eq(struct mthca_dev *dev, - struct mthca_eq *eq) -{ - struct mthca_mailbox *mailbox; - int err; - u8 status; - int npages = (eq->nent * MTHCA_EQ_ENTRY_SIZE + PAGE_SIZE - 1) / - PAGE_SIZE; - int i; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return; - - err = mthca_HW2SW_EQ(dev, mailbox, eq->eqn, &status); - if (err) - mthca_warn(dev, "HW2SW_EQ failed (%d)\n", err); - if (status) - mthca_warn(dev, "HW2SW_EQ returned status 0x%02x\n", status); - - dev->eq_table.arm_mask &= ~eq->eqn_mask; - - if (0) { - mthca_dbg(dev, "Dumping EQ context %02x:\n", eq->eqn); - for (i = 0; i < sizeof (struct mthca_eq_context) / 4; ++i) { - if (i % 4 == 0) - printk("[%02x] ", i * 4); - printk(" %08x", be32_to_cpup(mailbox->buf + i * 4)); - if ((i + 1) % 4 == 0) - printk("\n"); - } - } - - mthca_free_mr(dev, &eq->mr); - for (i = 0; i < npages; ++i) - pci_free_consistent(dev->pdev, PAGE_SIZE, - eq->page_list[i].buf, - pci_unmap_addr(&eq->page_list[i], mapping)); - - kfree(eq->page_list); - mthca_free_mailbox(dev, mailbox); -} - -static void mthca_free_irqs(struct mthca_dev *dev) -{ - int i; - - if (dev->eq_table.have_irq) - free_irq(dev->pdev->irq, dev); - for (i = 0; i < MTHCA_NUM_EQ; ++i) - if (dev->eq_table.eq[i].have_irq) { - free_irq(dev->eq_table.eq[i].msi_x_vector, - dev->eq_table.eq + i); - dev->eq_table.eq[i].have_irq = 0; - } -} - -static int mthca_map_reg(struct mthca_dev *dev, - unsigned long offset, unsigned long size, - void __iomem **map) -{ - unsigned long base = pci_resource_start(dev->pdev, 0); - - *map = ioremap(base + offset, size); - if (!*map) - return -ENOMEM; - - return 0; -} - -static int mthca_map_eq_regs(struct mthca_dev *dev) -{ - if (mthca_is_memfree(dev)) { - /* - * We assume that the EQ arm and EQ set CI registers - * fall within the first BAR. We can't trust the - * values firmware gives us, since those addresses are - * valid on the HCA's side of the PCI bus but not - * necessarily the host side. - */ - if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.clr_int_base, MTHCA_CLR_INT_SIZE, - &dev->clr_base)) { - mthca_err(dev, "Couldn't map interrupt clear register, " - "aborting.\n"); - return -ENOMEM; - } - - /* - * Add 4 because we limit ourselves to EQs 0 ... 31, - * so we only need the low word of the register. - */ - if (mthca_map_reg(dev, ((pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.eq_arm_base) + 4, 4, - &dev->eq_regs.arbel.eq_arm)) { - mthca_err(dev, "Couldn't map EQ arm register, aborting.\n"); - iounmap(dev->clr_base); - return -ENOMEM; - } - - if (mthca_map_reg(dev, (pci_resource_len(dev->pdev, 0) - 1) & - dev->fw.arbel.eq_set_ci_base, - MTHCA_EQ_SET_CI_SIZE, - &dev->eq_regs.arbel.eq_set_ci_base)) { - mthca_err(dev, "Couldn't map EQ CI register, aborting.\n"); - iounmap(dev->eq_regs.arbel.eq_arm); - iounmap(dev->clr_base); - return -ENOMEM; - } - } else { - if (mthca_map_reg(dev, MTHCA_CLR_INT_BASE, MTHCA_CLR_INT_SIZE, - &dev->clr_base)) { - mthca_err(dev, "Couldn't map interrupt clear register, " - "aborting.\n"); - return -ENOMEM; - } - - if (mthca_map_reg(dev, MTHCA_ECR_BASE, - MTHCA_ECR_SIZE + MTHCA_ECR_CLR_SIZE, - &dev->eq_regs.tavor.ecr_base)) { - mthca_err(dev, "Couldn't map ecr register, " - "aborting.\n"); - iounmap(dev->clr_base); - return -ENOMEM; - } - } - - return 0; - -} - -static void mthca_unmap_eq_regs(struct mthca_dev *dev) -{ - if (mthca_is_memfree(dev)) { - iounmap(dev->eq_regs.arbel.eq_set_ci_base); - iounmap(dev->eq_regs.arbel.eq_arm); - iounmap(dev->clr_base); - } else { - iounmap(dev->eq_regs.tavor.ecr_base); - iounmap(dev->clr_base); - } -} - -int mthca_map_eq_icm(struct mthca_dev *dev, u64 icm_virt) -{ - int ret; - u8 status; - - /* - * We assume that mapping one page is enough for the whole EQ - * context table. This is fine with all current HCAs, because - * we only use 32 EQs and each EQ uses 32 bytes of context - * memory, or 1 KB total. - */ - dev->eq_table.icm_virt = icm_virt; - dev->eq_table.icm_page = alloc_page(GFP_HIGHUSER); - if (!dev->eq_table.icm_page) - return -ENOMEM; - dev->eq_table.icm_dma = pci_map_page(dev->pdev, dev->eq_table.icm_page, 0, - PAGE_SIZE, PCI_DMA_BIDIRECTIONAL); - if (pci_dma_mapping_error(dev->pdev, dev->eq_table.icm_dma)) { - __free_page(dev->eq_table.icm_page); - return -ENOMEM; - } - - ret = mthca_MAP_ICM_page(dev, dev->eq_table.icm_dma, icm_virt, &status); - if (!ret && status) - ret = -EINVAL; - if (ret) { - pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - __free_page(dev->eq_table.icm_page); - } - - return ret; -} - -void mthca_unmap_eq_icm(struct mthca_dev *dev) -{ - u8 status; - - mthca_UNMAP_ICM(dev, dev->eq_table.icm_virt, 1, &status); - pci_unmap_page(dev->pdev, dev->eq_table.icm_dma, PAGE_SIZE, - PCI_DMA_BIDIRECTIONAL); - __free_page(dev->eq_table.icm_page); -} - -int mthca_init_eq_table(struct mthca_dev *dev) -{ - int err; - u8 status; - u8 intr; - int i; - - err = mthca_alloc_init(&dev->eq_table.alloc, - dev->limits.num_eqs, - dev->limits.num_eqs - 1, - dev->limits.reserved_eqs); - if (err) - return err; - - err = mthca_map_eq_regs(dev); - if (err) - goto err_out_free; - - if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { - dev->eq_table.clr_mask = 0; - } else { - dev->eq_table.clr_mask = - swab32(1 << (dev->eq_table.inta_pin & 31)); - dev->eq_table.clr_int = dev->clr_base + - (dev->eq_table.inta_pin < 32 ? 4 : 0); - } - - dev->eq_table.arm_mask = 0; - - intr = dev->eq_table.inta_pin; - - err = mthca_create_eq(dev, dev->limits.num_cqs + MTHCA_NUM_SPARE_EQE, - (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 128 : intr, - &dev->eq_table.eq[MTHCA_EQ_COMP]); - if (err) - goto err_out_unmap; - - err = mthca_create_eq(dev, MTHCA_NUM_ASYNC_EQE + MTHCA_NUM_SPARE_EQE, - (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 129 : intr, - &dev->eq_table.eq[MTHCA_EQ_ASYNC]); - if (err) - goto err_out_comp; - - err = mthca_create_eq(dev, MTHCA_NUM_CMD_EQE + MTHCA_NUM_SPARE_EQE, - (dev->mthca_flags & MTHCA_FLAG_MSI_X) ? 130 : intr, - &dev->eq_table.eq[MTHCA_EQ_CMD]); - if (err) - goto err_out_async; - - if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { - static const char *eq_name[] = { - [MTHCA_EQ_COMP] = DRV_NAME " (comp)", - [MTHCA_EQ_ASYNC] = DRV_NAME " (async)", - [MTHCA_EQ_CMD] = DRV_NAME " (cmd)" - }; - - for (i = 0; i < MTHCA_NUM_EQ; ++i) { - err = request_irq(dev->eq_table.eq[i].msi_x_vector, - mthca_is_memfree(dev) ? - mthca_arbel_msi_x_interrupt : - mthca_tavor_msi_x_interrupt, - 0, eq_name[i], dev->eq_table.eq + i); - if (err) - goto err_out_cmd; - dev->eq_table.eq[i].have_irq = 1; - } - } else { - err = request_irq(dev->pdev->irq, - mthca_is_memfree(dev) ? - mthca_arbel_interrupt : - mthca_tavor_interrupt, - IRQF_SHARED, DRV_NAME, dev); - if (err) - goto err_out_cmd; - dev->eq_table.have_irq = 1; - } - - err = mthca_MAP_EQ(dev, async_mask(dev), - 0, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status); - if (err) - mthca_warn(dev, "MAP_EQ for async EQ %d failed (%d)\n", - dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, err); - if (status) - mthca_warn(dev, "MAP_EQ for async EQ %d returned status 0x%02x\n", - dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, status); - - err = mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK, - 0, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status); - if (err) - mthca_warn(dev, "MAP_EQ for cmd EQ %d failed (%d)\n", - dev->eq_table.eq[MTHCA_EQ_CMD].eqn, err); - if (status) - mthca_warn(dev, "MAP_EQ for cmd EQ %d returned status 0x%02x\n", - dev->eq_table.eq[MTHCA_EQ_CMD].eqn, status); - - for (i = 0; i < MTHCA_NUM_EQ; ++i) - if (mthca_is_memfree(dev)) - arbel_eq_req_not(dev, dev->eq_table.eq[i].eqn_mask); - else - tavor_eq_req_not(dev, dev->eq_table.eq[i].eqn); - - return 0; - -err_out_cmd: - mthca_free_irqs(dev); - mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_CMD]); - -err_out_async: - mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_ASYNC]); - -err_out_comp: - mthca_free_eq(dev, &dev->eq_table.eq[MTHCA_EQ_COMP]); - -err_out_unmap: - mthca_unmap_eq_regs(dev); - -err_out_free: - mthca_alloc_cleanup(&dev->eq_table.alloc); - return err; -} - -void mthca_cleanup_eq_table(struct mthca_dev *dev) -{ - u8 status; - int i; - - mthca_free_irqs(dev); - - mthca_MAP_EQ(dev, async_mask(dev), - 1, dev->eq_table.eq[MTHCA_EQ_ASYNC].eqn, &status); - mthca_MAP_EQ(dev, MTHCA_CMD_EVENT_MASK, - 1, dev->eq_table.eq[MTHCA_EQ_CMD].eqn, &status); - - for (i = 0; i < MTHCA_NUM_EQ; ++i) - mthca_free_eq(dev, &dev->eq_table.eq[i]); - - mthca_unmap_eq_regs(dev); - - mthca_alloc_cleanup(&dev->eq_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mad.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_mad.c deleted file mode 100644 index 5648659ff0b0..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mad.c +++ /dev/null @@ -1,346 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" - -enum { - MTHCA_VENDOR_CLASS1 = 0x9, - MTHCA_VENDOR_CLASS2 = 0xa -}; - -static int mthca_update_rate(struct mthca_dev *dev, u8 port_num) -{ - struct ib_port_attr *tprops = NULL; - int ret; - - tprops = kmalloc(sizeof *tprops, GFP_KERNEL); - if (!tprops) - return -ENOMEM; - - ret = ib_query_port(&dev->ib_dev, port_num, tprops); - if (ret) { - printk(KERN_WARNING "ib_query_port failed (%d) for %s port %d\n", - ret, dev->ib_dev.name, port_num); - goto out; - } - - dev->rate[port_num - 1] = tprops->active_speed * - ib_width_enum_to_int(tprops->active_width); - -out: - kfree(tprops); - return ret; -} - -static void update_sm_ah(struct mthca_dev *dev, - u8 port_num, u16 lid, u8 sl) -{ - struct ib_ah *new_ah; - struct ib_ah_attr ah_attr; - unsigned long flags; - - if (!dev->send_agent[port_num - 1][0]) - return; - - memset(&ah_attr, 0, sizeof ah_attr); - ah_attr.dlid = lid; - ah_attr.sl = sl; - ah_attr.port_num = port_num; - - new_ah = ib_create_ah(dev->send_agent[port_num - 1][0]->qp->pd, - &ah_attr); - if (IS_ERR(new_ah)) - return; - - spin_lock_irqsave(&dev->sm_lock, flags); - if (dev->sm_ah[port_num - 1]) - ib_destroy_ah(dev->sm_ah[port_num - 1]); - dev->sm_ah[port_num - 1] = new_ah; - spin_unlock_irqrestore(&dev->sm_lock, flags); -} - -/* - * Snoop SM MADs for port info and P_Key table sets, so we can - * synthesize LID change and P_Key change events. - */ -static void smp_snoop(struct ib_device *ibdev, - u8 port_num, - struct ib_mad *mad, - u16 prev_lid) -{ - struct ib_event event; - - if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - mad->mad_hdr.method == IB_MGMT_METHOD_SET) { - if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO) { - struct ib_port_info *pinfo = - (struct ib_port_info *) ((struct ib_smp *) mad)->data; - u16 lid = be16_to_cpu(pinfo->lid); - - mthca_update_rate(to_mdev(ibdev), port_num); - update_sm_ah(to_mdev(ibdev), port_num, - be16_to_cpu(pinfo->sm_lid), - pinfo->neighbormtu_mastersmsl & 0xf); - - event.device = ibdev; - event.element.port_num = port_num; - - if (pinfo->clientrereg_resv_subnetto & 0x80) { - event.event = IB_EVENT_CLIENT_REREGISTER; - ib_dispatch_event(&event); - } - - if (prev_lid != lid) { - event.event = IB_EVENT_LID_CHANGE; - ib_dispatch_event(&event); - } - } - - if (mad->mad_hdr.attr_id == IB_SMP_ATTR_PKEY_TABLE) { - event.device = ibdev; - event.event = IB_EVENT_PKEY_CHANGE; - event.element.port_num = port_num; - ib_dispatch_event(&event); - } - } -} - -static void node_desc_override(struct ib_device *dev, - struct ib_mad *mad) -{ - if ((mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - mad->mad_hdr.method == IB_MGMT_METHOD_GET_RESP && - mad->mad_hdr.attr_id == IB_SMP_ATTR_NODE_DESC) { - mutex_lock(&to_mdev(dev)->cap_mask_mutex); - memcpy(((struct ib_smp *) mad)->data, dev->node_desc, 64); - mutex_unlock(&to_mdev(dev)->cap_mask_mutex); - } -} - -static void forward_trap(struct mthca_dev *dev, - u8 port_num, - struct ib_mad *mad) -{ - int qpn = mad->mad_hdr.mgmt_class != IB_MGMT_CLASS_SUBN_LID_ROUTED; - struct ib_mad_send_buf *send_buf; - struct ib_mad_agent *agent = dev->send_agent[port_num - 1][qpn]; - int ret; - unsigned long flags; - - if (agent) { - send_buf = ib_create_send_mad(agent, qpn, 0, 0, IB_MGMT_MAD_HDR, - IB_MGMT_MAD_DATA, GFP_ATOMIC); - /* - * We rely here on the fact that MLX QPs don't use the - * address handle after the send is posted (this is - * wrong following the IB spec strictly, but we know - * it's OK for our devices). - */ - spin_lock_irqsave(&dev->sm_lock, flags); - memcpy(send_buf->mad, mad, sizeof *mad); - if ((send_buf->ah = dev->sm_ah[port_num - 1])) - ret = ib_post_send_mad(send_buf, NULL); - else - ret = -EINVAL; - spin_unlock_irqrestore(&dev->sm_lock, flags); - - if (ret) - ib_free_send_mad(send_buf); - } -} - -int mthca_process_mad(struct ib_device *ibdev, - int mad_flags, - u8 port_num, - struct ib_wc *in_wc, - struct ib_grh *in_grh, - struct ib_mad *in_mad, - struct ib_mad *out_mad) -{ - int err; - u8 status; - u16 slid = in_wc ? in_wc->slid : be16_to_cpu(IB_LID_PERMISSIVE); - u16 prev_lid = 0; - struct ib_port_attr pattr; - - /* Forward locally generated traps to the SM */ - if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP && - slid == 0) { - forward_trap(to_mdev(ibdev), port_num, in_mad); - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; - } - - /* - * Only handle SM gets, sets and trap represses for SM class - * - * Only handle PMA and Mellanox vendor-specific class gets and - * sets for other classes. - */ - if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) { - if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_SET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_TRAP_REPRESS) - return IB_MAD_RESULT_SUCCESS; - - /* - * Don't process SMInfo queries or vendor-specific - * MADs -- the SMA can't handle them. - */ - if (in_mad->mad_hdr.attr_id == IB_SMP_ATTR_SM_INFO || - ((in_mad->mad_hdr.attr_id & IB_SMP_ATTR_VENDOR_MASK) == - IB_SMP_ATTR_VENDOR_MASK)) - return IB_MAD_RESULT_SUCCESS; - } else if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT || - in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS1 || - in_mad->mad_hdr.mgmt_class == MTHCA_VENDOR_CLASS2) { - if (in_mad->mad_hdr.method != IB_MGMT_METHOD_GET && - in_mad->mad_hdr.method != IB_MGMT_METHOD_SET) - return IB_MAD_RESULT_SUCCESS; - } else - return IB_MAD_RESULT_SUCCESS; - if ((in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED || - in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) && - in_mad->mad_hdr.method == IB_MGMT_METHOD_SET && - in_mad->mad_hdr.attr_id == IB_SMP_ATTR_PORT_INFO && - !ib_query_port(ibdev, port_num, &pattr)) - prev_lid = pattr.lid; - - err = mthca_MAD_IFC(to_mdev(ibdev), - mad_flags & IB_MAD_IGNORE_MKEY, - mad_flags & IB_MAD_IGNORE_BKEY, - port_num, in_wc, in_grh, in_mad, out_mad, - &status); - if (err) { - mthca_err(to_mdev(ibdev), "MAD_IFC failed\n"); - return IB_MAD_RESULT_FAILURE; - } - if (status == MTHCA_CMD_STAT_BAD_PKT) - return IB_MAD_RESULT_SUCCESS; - if (status) { - mthca_err(to_mdev(ibdev), "MAD_IFC returned status %02x\n", - status); - return IB_MAD_RESULT_FAILURE; - } - - if (!out_mad->mad_hdr.status) { - smp_snoop(ibdev, port_num, in_mad, prev_lid); - node_desc_override(ibdev, out_mad); - } - - /* set return bit in status of directed route responses */ - if (in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) - out_mad->mad_hdr.status |= cpu_to_be16(1 << 15); - - if (in_mad->mad_hdr.method == IB_MGMT_METHOD_TRAP_REPRESS) - /* no response for trap repress */ - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; - - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; -} - -static void send_handler(struct ib_mad_agent *agent, - struct ib_mad_send_wc *mad_send_wc) -{ - ib_free_send_mad(mad_send_wc->send_buf); -} - -int mthca_create_agents(struct mthca_dev *dev) -{ - struct ib_mad_agent *agent; - int p, q; - int ret; - - spin_lock_init(&dev->sm_lock); - - for (p = 0; p < dev->limits.num_ports; ++p) - for (q = 0; q <= 1; ++q) { - agent = ib_register_mad_agent(&dev->ib_dev, p + 1, - q ? IB_QPT_GSI : IB_QPT_SMI, - NULL, 0, send_handler, - NULL, NULL); - if (IS_ERR(agent)) { - ret = PTR_ERR(agent); - goto err; - } - dev->send_agent[p][q] = agent; - } - - - for (p = 1; p <= dev->limits.num_ports; ++p) { - ret = mthca_update_rate(dev, p); - if (ret) { - mthca_err(dev, "Failed to obtain port %d rate." - " aborting.\n", p); - goto err; - } - } - - return 0; - -err: - for (p = 0; p < dev->limits.num_ports; ++p) - for (q = 0; q <= 1; ++q) - if (dev->send_agent[p][q]) - ib_unregister_mad_agent(dev->send_agent[p][q]); - - return ret; -} - -void mthca_free_agents(struct mthca_dev *dev) -{ - struct ib_mad_agent *agent; - int p, q; - - for (p = 0; p < dev->limits.num_ports; ++p) { - for (q = 0; q <= 1; ++q) { - agent = dev->send_agent[p][q]; - dev->send_agent[p][q] = NULL; - ib_unregister_mad_agent(agent); - } - - if (dev->sm_ah[p]) - ib_destroy_ah(dev->sm_ah[p]); - } -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c deleted file mode 100644 index c7e9851e70e0..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c +++ /dev/null @@ -1,1363 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define LINUXKPI_PARAM_PREFIX mthca_ - -#include -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_config_reg.h" -#include "mthca_cmd.h" -#include "mthca_profile.h" -#include "mthca_memfree.h" -#include "mthca_wqe.h" - -MODULE_AUTHOR("Roland Dreier"); -MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver"); -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_VERSION(mthca, 1); -MODULE_DEPEND(mthca, linuxkpi, 1, 1, 1); -MODULE_DEPEND(mthca, ibcore, 1, 1, 1); - -#ifdef CONFIG_INFINIBAND_MTHCA_DEBUG - -int mthca_debug_level = 0; -module_param_named(debug_level, mthca_debug_level, int, 0644); -MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0"); - -#endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */ - -#ifdef CONFIG_PCI_MSI - -static int msi_x = 1; -module_param(msi_x, int, 0444); -MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero"); - -#else /* CONFIG_PCI_MSI */ - -#define msi_x (0) - -#endif /* CONFIG_PCI_MSI */ - -static int tune_pci = 0; -module_param(tune_pci, int, 0444); -MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero"); - -DEFINE_MUTEX(mthca_device_mutex); - -#define MTHCA_DEFAULT_NUM_QP (1 << 16) -#define MTHCA_DEFAULT_RDB_PER_QP (1 << 2) -#define MTHCA_DEFAULT_NUM_CQ (1 << 16) -#define MTHCA_DEFAULT_NUM_MCG (1 << 13) -#define MTHCA_DEFAULT_NUM_MPT (1 << 17) -#define MTHCA_DEFAULT_NUM_MTT (1 << 20) -#define MTHCA_DEFAULT_NUM_UDAV (1 << 15) -#define MTHCA_DEFAULT_NUM_RESERVED_MTTS (1 << 18) -#define MTHCA_DEFAULT_NUM_UARC_SIZE (1 << 18) - -static struct mthca_profile hca_profile = { - .num_qp = MTHCA_DEFAULT_NUM_QP, - .rdb_per_qp = MTHCA_DEFAULT_RDB_PER_QP, - .num_cq = MTHCA_DEFAULT_NUM_CQ, - .num_mcg = MTHCA_DEFAULT_NUM_MCG, - .num_mpt = MTHCA_DEFAULT_NUM_MPT, - .num_mtt = MTHCA_DEFAULT_NUM_MTT, - .num_udav = MTHCA_DEFAULT_NUM_UDAV, /* Tavor only */ - .fmr_reserved_mtts = MTHCA_DEFAULT_NUM_RESERVED_MTTS, /* Tavor only */ - .uarc_size = MTHCA_DEFAULT_NUM_UARC_SIZE, /* Arbel only */ -}; - -module_param_named(num_qp, hca_profile.num_qp, int, 0444); -MODULE_PARM_DESC(num_qp, "maximum number of QPs per HCA"); - -module_param_named(rdb_per_qp, hca_profile.rdb_per_qp, int, 0444); -MODULE_PARM_DESC(rdb_per_qp, "number of RDB buffers per QP"); - -module_param_named(num_cq, hca_profile.num_cq, int, 0444); -MODULE_PARM_DESC(num_cq, "maximum number of CQs per HCA"); - -module_param_named(num_mcg, hca_profile.num_mcg, int, 0444); -MODULE_PARM_DESC(num_mcg, "maximum number of multicast groups per HCA"); - -module_param_named(num_mpt, hca_profile.num_mpt, int, 0444); -MODULE_PARM_DESC(num_mpt, - "maximum number of memory protection table entries per HCA"); - -module_param_named(num_mtt, hca_profile.num_mtt, int, 0444); -MODULE_PARM_DESC(num_mtt, - "maximum number of memory translation table segments per HCA"); - -module_param_named(num_udav, hca_profile.num_udav, int, 0444); -MODULE_PARM_DESC(num_udav, "maximum number of UD address vectors per HCA"); - -module_param_named(fmr_reserved_mtts, hca_profile.fmr_reserved_mtts, int, 0444); -MODULE_PARM_DESC(fmr_reserved_mtts, - "number of memory translation table segments reserved for FMR"); - -static int log_mtts_per_seg; -module_param_named(log_mtts_per_seg, log_mtts_per_seg, int, 0444); -MODULE_PARM_DESC(log_mtts_per_seg, "Log2 number of MTT entries per segment (1-5)"); - -static char mthca_version[] __devinitdata = - DRV_NAME ": Mellanox InfiniBand HCA driver v" - DRV_VERSION " (" DRV_RELDATE ")\n"; - -static int mthca_tune_pci(struct mthca_dev *mdev) -{ - if (!tune_pci) - return 0; - - /* First try to max out Read Byte Count */ - if (pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX)) { - if (pcix_set_mmrbc(mdev->pdev, pcix_get_max_mmrbc(mdev->pdev))) { - mthca_err(mdev, "Couldn't set PCI-X max read count, " - "aborting.\n"); - return -ENODEV; - } - } else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE)) - mthca_info(mdev, "No PCI-X capability, not setting RBC.\n"); - - if (pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP)) { - if (pcie_set_readrq(mdev->pdev, 4096)) { - mthca_err(mdev, "Couldn't write PCI Express read request, " - "aborting.\n"); - return -ENODEV; - } - } else if (mdev->mthca_flags & MTHCA_FLAG_PCIE) - mthca_info(mdev, "No PCI Express capability, " - "not setting Max Read Request Size.\n"); - - return 0; -} - -static int mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim) -{ - int err; - u8 status; - - mdev->limits.mtt_seg_size = (1 << log_mtts_per_seg) * 8; - err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status); - if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); - return err; - } - if (status) { - mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - if (dev_lim->min_page_sz > PAGE_SIZE) { - mthca_err(mdev, "HCA minimum page size of %d bigger than " - "kernel PAGE_SIZE of %d, aborting.\n", - dev_lim->min_page_sz, PAGE_SIZE); - return -ENODEV; - } - if (dev_lim->num_ports > MTHCA_MAX_PORTS) { - mthca_err(mdev, "HCA has %d ports, but we only support %d, " - "aborting.\n", - dev_lim->num_ports, MTHCA_MAX_PORTS); - return -ENODEV; - } - - if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) { - mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than " - "PCI resource 2 size of 0x%llx, aborting.\n", - dev_lim->uar_size, - (unsigned long long)pci_resource_len(mdev->pdev, 2)); - return -ENODEV; - } - - mdev->limits.num_ports = dev_lim->num_ports; - mdev->limits.vl_cap = dev_lim->max_vl; - mdev->limits.mtu_cap = dev_lim->max_mtu; - mdev->limits.gid_table_len = dev_lim->max_gids; - mdev->limits.pkey_table_len = dev_lim->max_pkeys; - mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay; - /* - * Need to allow for worst case send WQE overhead and check - * whether max_desc_sz imposes a lower limit than max_sg; UD - * send has the biggest overhead. - */ - mdev->limits.max_sg = min_t(int, dev_lim->max_sg, - (dev_lim->max_desc_sz - - sizeof (struct mthca_next_seg) - - (mthca_is_memfree(mdev) ? - sizeof (struct mthca_arbel_ud_seg) : - sizeof (struct mthca_tavor_ud_seg))) / - sizeof (struct mthca_data_seg)); - mdev->limits.max_wqes = dev_lim->max_qp_sz; - mdev->limits.max_qp_init_rdma = dev_lim->max_requester_per_qp; - mdev->limits.reserved_qps = dev_lim->reserved_qps; - mdev->limits.max_srq_wqes = dev_lim->max_srq_sz; - mdev->limits.reserved_srqs = dev_lim->reserved_srqs; - mdev->limits.reserved_eecs = dev_lim->reserved_eecs; - mdev->limits.max_desc_sz = dev_lim->max_desc_sz; - mdev->limits.max_srq_sge = mthca_max_srq_sge(mdev); - /* - * Subtract 1 from the limit because we need to allocate a - * spare CQE so the HCA HW can tell the difference between an - * empty CQ and a full CQ. - */ - mdev->limits.max_cqes = dev_lim->max_cq_sz - 1; - mdev->limits.reserved_cqs = dev_lim->reserved_cqs; - mdev->limits.reserved_eqs = dev_lim->reserved_eqs; - mdev->limits.reserved_mtts = dev_lim->reserved_mtts; - mdev->limits.reserved_mrws = dev_lim->reserved_mrws; - mdev->limits.reserved_uars = dev_lim->reserved_uars; - mdev->limits.reserved_pds = dev_lim->reserved_pds; - mdev->limits.port_width_cap = dev_lim->max_port_width; - mdev->limits.page_size_cap = ~(u32) (dev_lim->min_page_sz - 1); - mdev->limits.flags = dev_lim->flags; - /* - * For old FW that doesn't return static rate support, use a - * value of 0x3 (only static rate values of 0 or 1 are handled), - * except on Sinai, where even old FW can handle static rate - * values of 2 and 3. - */ - if (dev_lim->stat_rate_support) - mdev->limits.stat_rate_support = dev_lim->stat_rate_support; - else if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - mdev->limits.stat_rate_support = 0xf; - else - mdev->limits.stat_rate_support = 0x3; - - /* IB_DEVICE_RESIZE_MAX_WR not supported by driver. - May be doable since hardware supports it for SRQ. - - IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver. - - IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not - supported by driver. */ - mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT | - IB_DEVICE_PORT_ACTIVE_EVENT | - IB_DEVICE_SYS_IMAGE_GUID | - IB_DEVICE_RC_RNR_NAK_GEN; - - if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR) - mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR; - - if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR) - mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR; - - if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI) - mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI; - - if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG) - mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG; - - if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE) - mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE; - - if (dev_lim->flags & DEV_LIM_FLAG_SRQ) - mdev->mthca_flags |= MTHCA_FLAG_SRQ; - - if (mthca_is_memfree(mdev)) - if (dev_lim->flags & DEV_LIM_FLAG_IPOIB_CSUM) - mdev->device_cap_flags |= IB_DEVICE_UD_IP_CSUM; - - return 0; -} - -static int mthca_init_tavor(struct mthca_dev *mdev) -{ - s64 size; - u8 status; - int err; - struct mthca_dev_lim dev_lim; - struct mthca_profile profile; - struct mthca_init_hca_param init_hca; - - err = mthca_SYS_EN(mdev, &status); - if (err) { - mthca_err(mdev, "SYS_EN command failed, aborting.\n"); - return err; - } - if (status) { - mthca_err(mdev, "SYS_EN returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - - err = mthca_QUERY_FW(mdev, &status); - if (err) { - mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); - goto err_disable; - } - if (status) { - mthca_err(mdev, "QUERY_FW returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_disable; - } - err = mthca_QUERY_DDR(mdev, &status); - if (err) { - mthca_err(mdev, "QUERY_DDR command failed, aborting.\n"); - goto err_disable; - } - if (status) { - mthca_err(mdev, "QUERY_DDR returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_disable; - } - - err = mthca_dev_lim(mdev, &dev_lim); - if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); - goto err_disable; - } - - profile = hca_profile; - profile.num_uar = dev_lim.uar_size / PAGE_SIZE; - profile.uarc_size = 0; - if (mdev->mthca_flags & MTHCA_FLAG_SRQ) - profile.num_srq = dev_lim.max_srqs; - - size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); - if (size < 0) { - err = size; - goto err_disable; - } - - err = mthca_INIT_HCA(mdev, &init_hca, &status); - if (err) { - mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); - goto err_disable; - } - if (status) { - mthca_err(mdev, "INIT_HCA returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_disable; - } - - return 0; - -err_disable: - mthca_SYS_DIS(mdev, &status); - - return err; -} - -static int mthca_load_fw(struct mthca_dev *mdev) -{ - u8 status; - int err; - - /* FIXME: use HCA-attached memory for FW if present */ - - mdev->fw.arbel.fw_icm = - mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages, - GFP_HIGHUSER | __GFP_NOWARN, 0); - if (!mdev->fw.arbel.fw_icm) { - mthca_err(mdev, "Couldn't allocate FW area, aborting.\n"); - return -ENOMEM; - } - - err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status); - if (err) { - mthca_err(mdev, "MAP_FA command failed, aborting.\n"); - goto err_free; - } - if (status) { - mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status); - err = -EINVAL; - goto err_free; - } - err = mthca_RUN_FW(mdev, &status); - if (err) { - mthca_err(mdev, "RUN_FW command failed, aborting.\n"); - goto err_unmap_fa; - } - if (status) { - mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status); - err = -EINVAL; - goto err_unmap_fa; - } - - return 0; - -err_unmap_fa: - mthca_UNMAP_FA(mdev, &status); - -err_free: - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); - return err; -} - -static int mthca_init_icm(struct mthca_dev *mdev, - struct mthca_dev_lim *dev_lim, - struct mthca_init_hca_param *init_hca, - u64 icm_size) -{ - u64 aux_pages; - u8 status; - int err; - - err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status); - if (err) { - mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n"); - return err; - } - if (status) { - mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - - mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n", - (unsigned long long) icm_size >> 10, - (unsigned long long) aux_pages << 2); - - mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages, - GFP_HIGHUSER | __GFP_NOWARN, 0); - if (!mdev->fw.arbel.aux_icm) { - mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n"); - return -ENOMEM; - } - - err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status); - if (err) { - mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n"); - goto err_free_aux; - } - if (status) { - mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status); - err = -EINVAL; - goto err_free_aux; - } - - err = mthca_map_eq_icm(mdev, init_hca->eqc_base); - if (err) { - mthca_err(mdev, "Failed to map EQ context memory, aborting.\n"); - goto err_unmap_aux; - } - - /* CPU writes to non-reserved MTTs, while HCA might DMA to reserved mtts */ - mdev->limits.reserved_mtts = ALIGN(mdev->limits.reserved_mtts * mdev->limits.mtt_seg_size, - dma_get_cache_alignment()) / mdev->limits.mtt_seg_size; - - mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base, - mdev->limits.mtt_seg_size, - mdev->limits.num_mtt_segs, - mdev->limits.reserved_mtts, - 1, 0); - if (!mdev->mr_table.mtt_table) { - mthca_err(mdev, "Failed to map MTT context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_eq; - } - - mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base, - dev_lim->mpt_entry_sz, - mdev->limits.num_mpts, - mdev->limits.reserved_mrws, - 1, 1); - if (!mdev->mr_table.mpt_table) { - mthca_err(mdev, "Failed to map MPT context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_mtt; - } - - mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base, - dev_lim->qpc_entry_sz, - mdev->limits.num_qps, - mdev->limits.reserved_qps, - 0, 0); - if (!mdev->qp_table.qp_table) { - mthca_err(mdev, "Failed to map QP context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_mpt; - } - - mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base, - dev_lim->eqpc_entry_sz, - mdev->limits.num_qps, - mdev->limits.reserved_qps, - 0, 0); - if (!mdev->qp_table.eqp_table) { - mthca_err(mdev, "Failed to map EQP context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_qp; - } - - mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base, - MTHCA_RDB_ENTRY_SIZE, - mdev->limits.num_qps << - mdev->qp_table.rdb_shift, 0, - 0, 0); - if (!mdev->qp_table.rdb_table) { - mthca_err(mdev, "Failed to map RDB context memory, aborting\n"); - err = -ENOMEM; - goto err_unmap_eqp; - } - - mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base, - dev_lim->cqc_entry_sz, - mdev->limits.num_cqs, - mdev->limits.reserved_cqs, - 0, 0); - if (!mdev->cq_table.table) { - mthca_err(mdev, "Failed to map CQ context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_rdb; - } - - if (mdev->mthca_flags & MTHCA_FLAG_SRQ) { - mdev->srq_table.table = - mthca_alloc_icm_table(mdev, init_hca->srqc_base, - dev_lim->srq_entry_sz, - mdev->limits.num_srqs, - mdev->limits.reserved_srqs, - 0, 0); - if (!mdev->srq_table.table) { - mthca_err(mdev, "Failed to map SRQ context memory, " - "aborting.\n"); - err = -ENOMEM; - goto err_unmap_cq; - } - } - - /* - * It's not strictly required, but for simplicity just map the - * whole multicast group table now. The table isn't very big - * and it's a lot easier than trying to track ref counts. - */ - mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base, - MTHCA_MGM_ENTRY_SIZE, - mdev->limits.num_mgms + - mdev->limits.num_amgms, - mdev->limits.num_mgms + - mdev->limits.num_amgms, - 0, 0); - if (!mdev->mcg_table.table) { - mthca_err(mdev, "Failed to map MCG context memory, aborting.\n"); - err = -ENOMEM; - goto err_unmap_srq; - } - - return 0; - -err_unmap_srq: - if (mdev->mthca_flags & MTHCA_FLAG_SRQ) - mthca_free_icm_table(mdev, mdev->srq_table.table); - -err_unmap_cq: - mthca_free_icm_table(mdev, mdev->cq_table.table); - -err_unmap_rdb: - mthca_free_icm_table(mdev, mdev->qp_table.rdb_table); - -err_unmap_eqp: - mthca_free_icm_table(mdev, mdev->qp_table.eqp_table); - -err_unmap_qp: - mthca_free_icm_table(mdev, mdev->qp_table.qp_table); - -err_unmap_mpt: - mthca_free_icm_table(mdev, mdev->mr_table.mpt_table); - -err_unmap_mtt: - mthca_free_icm_table(mdev, mdev->mr_table.mtt_table); - -err_unmap_eq: - mthca_unmap_eq_icm(mdev); - -err_unmap_aux: - mthca_UNMAP_ICM_AUX(mdev, &status); - -err_free_aux: - mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); - - return err; -} - -static void mthca_free_icms(struct mthca_dev *mdev) -{ - u8 status; - - mthca_free_icm_table(mdev, mdev->mcg_table.table); - if (mdev->mthca_flags & MTHCA_FLAG_SRQ) - mthca_free_icm_table(mdev, mdev->srq_table.table); - mthca_free_icm_table(mdev, mdev->cq_table.table); - mthca_free_icm_table(mdev, mdev->qp_table.rdb_table); - mthca_free_icm_table(mdev, mdev->qp_table.eqp_table); - mthca_free_icm_table(mdev, mdev->qp_table.qp_table); - mthca_free_icm_table(mdev, mdev->mr_table.mpt_table); - mthca_free_icm_table(mdev, mdev->mr_table.mtt_table); - mthca_unmap_eq_icm(mdev); - - mthca_UNMAP_ICM_AUX(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.aux_icm, 0); -} - -static int mthca_init_arbel(struct mthca_dev *mdev) -{ - struct mthca_dev_lim dev_lim; - struct mthca_profile profile; - struct mthca_init_hca_param init_hca; - s64 icm_size; - u8 status; - int err; - - err = mthca_QUERY_FW(mdev, &status); - if (err) { - mthca_err(mdev, "QUERY_FW command failed, aborting.\n"); - return err; - } - if (status) { - mthca_err(mdev, "QUERY_FW returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - - err = mthca_ENABLE_LAM(mdev, &status); - if (err) { - mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n"); - return err; - } - if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) { - mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n"); - mdev->mthca_flags |= MTHCA_FLAG_NO_LAM; - } else if (status) { - mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, " - "aborting.\n", status); - return -EINVAL; - } - - err = mthca_load_fw(mdev); - if (err) { - mthca_err(mdev, "Failed to start FW, aborting.\n"); - goto err_disable; - } - - err = mthca_dev_lim(mdev, &dev_lim); - if (err) { - mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n"); - goto err_stop_fw; - } - - profile = hca_profile; - profile.num_uar = dev_lim.uar_size / PAGE_SIZE; - profile.num_udav = 0; - if (mdev->mthca_flags & MTHCA_FLAG_SRQ) - profile.num_srq = dev_lim.max_srqs; - - icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca); - if (icm_size < 0) { - err = icm_size; - goto err_stop_fw; - } - - err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size); - if (err) - goto err_stop_fw; - - err = mthca_INIT_HCA(mdev, &init_hca, &status); - if (err) { - mthca_err(mdev, "INIT_HCA command failed, aborting.\n"); - goto err_free_icm; - } - if (status) { - mthca_err(mdev, "INIT_HCA returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_free_icm; - } - - return 0; - -err_free_icm: - mthca_free_icms(mdev); - -err_stop_fw: - mthca_UNMAP_FA(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); - -err_disable: - if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) - mthca_DISABLE_LAM(mdev, &status); - - return err; -} - -static void mthca_close_hca(struct mthca_dev *mdev) -{ - u8 status; - - mthca_CLOSE_HCA(mdev, 0, &status); - - if (mthca_is_memfree(mdev)) { - mthca_free_icms(mdev); - - mthca_UNMAP_FA(mdev, &status); - mthca_free_icm(mdev, mdev->fw.arbel.fw_icm, 0); - - if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM)) - mthca_DISABLE_LAM(mdev, &status); - } else - mthca_SYS_DIS(mdev, &status); -} - -static int mthca_init_hca(struct mthca_dev *mdev) -{ - u8 status; - int err; - struct mthca_adapter adapter; - - if (mthca_is_memfree(mdev)) - err = mthca_init_arbel(mdev); - else - err = mthca_init_tavor(mdev); - - if (err) - return err; - - err = mthca_QUERY_ADAPTER(mdev, &adapter, &status); - if (err) { - mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n"); - goto err_close; - } - if (status) { - mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, " - "aborting.\n", status); - err = -EINVAL; - goto err_close; - } - - mdev->eq_table.inta_pin = adapter.inta_pin; - if (!mthca_is_memfree(mdev)) - mdev->rev_id = adapter.revision_id; - memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id); - - return 0; - -err_close: - mthca_close_hca(mdev); - return err; -} - -static int mthca_setup_hca(struct mthca_dev *dev) -{ - int err; - u8 status; - - MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock); - - err = mthca_init_uar_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "user access region table, aborting.\n"); - return err; - } - - err = mthca_uar_alloc(dev, &dev->driver_uar); - if (err) { - mthca_err(dev, "Failed to allocate driver access region, " - "aborting.\n"); - goto err_uar_table_free; - } - - dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE); - if (!dev->kar) { - mthca_err(dev, "Couldn't map kernel access region, " - "aborting.\n"); - err = -ENOMEM; - goto err_uar_free; - } - - err = mthca_init_pd_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "protection domain table, aborting.\n"); - goto err_kar_unmap; - } - - err = mthca_init_mr_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "memory region table, aborting.\n"); - goto err_pd_table_free; - } - - err = mthca_pd_alloc(dev, 1, &dev->driver_pd); - if (err) { - mthca_err(dev, "Failed to create driver PD, " - "aborting.\n"); - goto err_mr_table_free; - } - - err = mthca_init_eq_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "event queue table, aborting.\n"); - goto err_pd_free; - } - - err = mthca_cmd_use_events(dev); - if (err) { - mthca_err(dev, "Failed to switch to event-driven " - "firmware commands, aborting.\n"); - goto err_eq_table_free; - } - - err = mthca_NOP(dev, &status); - if (err || status) { - if (dev->mthca_flags & MTHCA_FLAG_MSI_X) { - mthca_warn(dev, "NOP command failed to generate interrupt " - "(IRQ %d).\n", - dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector); - mthca_warn(dev, "Trying again with MSI-X disabled.\n"); - } else { - mthca_err(dev, "NOP command failed to generate interrupt " - "(IRQ %d), aborting.\n", - dev->pdev->irq); - mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n"); - } - - goto err_cmd_poll; - } - - mthca_dbg(dev, "NOP command IRQ test passed\n"); - - err = mthca_init_cq_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "completion queue table, aborting.\n"); - goto err_cmd_poll; - } - - err = mthca_init_srq_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "shared receive queue table, aborting.\n"); - goto err_cq_table_free; - } - - err = mthca_init_qp_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "queue pair table, aborting.\n"); - goto err_srq_table_free; - } - - err = mthca_init_av_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "address vector table, aborting.\n"); - goto err_qp_table_free; - } - - err = mthca_init_mcg_table(dev); - if (err) { - mthca_err(dev, "Failed to initialize " - "multicast group table, aborting.\n"); - goto err_av_table_free; - } - - return 0; - -err_av_table_free: - mthca_cleanup_av_table(dev); - -err_qp_table_free: - mthca_cleanup_qp_table(dev); - -err_srq_table_free: - mthca_cleanup_srq_table(dev); - -err_cq_table_free: - mthca_cleanup_cq_table(dev); - -err_cmd_poll: - mthca_cmd_use_polling(dev); - -err_eq_table_free: - mthca_cleanup_eq_table(dev); - -err_pd_free: - mthca_pd_free(dev, &dev->driver_pd); - -err_mr_table_free: - mthca_cleanup_mr_table(dev); - -err_pd_table_free: - mthca_cleanup_pd_table(dev); - -err_kar_unmap: - iounmap(dev->kar); - -err_uar_free: - mthca_uar_free(dev, &dev->driver_uar); - -err_uar_table_free: - mthca_cleanup_uar_table(dev); - return err; -} - -static int mthca_enable_msi_x(struct mthca_dev *mdev) -{ - struct msix_entry entries[3]; - int err; - - entries[0].entry = 0; - entries[1].entry = 1; - entries[2].entry = 2; - - err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries)); - if (err) { - if (err > 0) - mthca_info(mdev, "Only %d MSI-X vectors available, " - "not using MSI-X\n", err); - return err; - } - - mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector; - mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector; - mdev->eq_table.eq[MTHCA_EQ_CMD ].msi_x_vector = entries[2].vector; - - return 0; -} - -/* Types of supported HCA */ -enum { - TAVOR, /* MT23108 */ - ARBEL_COMPAT, /* MT25208 in Tavor compat mode */ - ARBEL_NATIVE, /* MT25208 with extended features */ - SINAI /* MT25204 */ -}; - -#define MTHCA_FW_VER(major, minor, subminor) \ - (((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor)) - -static struct { - u64 latest_fw; - u32 flags; -} mthca_hca_table[] = { - [TAVOR] = { .latest_fw = MTHCA_FW_VER(3, 5, 0), - .flags = 0 }, - [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 8, 200), - .flags = MTHCA_FLAG_PCIE }, - [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 3, 0), - .flags = MTHCA_FLAG_MEMFREE | - MTHCA_FLAG_PCIE }, - [SINAI] = { .latest_fw = MTHCA_FW_VER(1, 2, 0), - .flags = MTHCA_FLAG_MEMFREE | - MTHCA_FLAG_PCIE | - MTHCA_FLAG_SINAI_OPT } -}; - -static int __mthca_init_one(struct pci_dev *pdev, int hca_type) -{ - int ddr_hidden = 0; - int err; - struct mthca_dev *mdev; - - printk(KERN_INFO PFX "Initializing %s\n", - pci_name(pdev)); - - err = pci_enable_device(pdev); - if (err) { - dev_err(&pdev->dev, "Cannot enable PCI device, " - "aborting.\n"); - return err; - } - - /* - * Check for BARs. We expect 0: 1MB, 2: 8MB, 4: DDR (may not - * be present) - */ - if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) || - pci_resource_len(pdev, 0) != 1 << 20) { - dev_err(&pdev->dev, "Missing DCS, aborting.\n"); - err = -ENODEV; - goto err_disable_pdev; - } - if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) { - dev_err(&pdev->dev, "Missing UAR, aborting.\n"); - err = -ENODEV; - goto err_disable_pdev; - } - if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM)) - ddr_hidden = 1; - - err = pci_request_regions(pdev, DRV_NAME); - if (err) { - dev_err(&pdev->dev, "Cannot obtain PCI resources, " - "aborting.\n"); - goto err_disable_pdev; - } - - pci_set_master(pdev); - - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n"); - err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n"); - goto err_free_res; - } - } - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); - if (err) { - dev_warn(&pdev->dev, "Warning: couldn't set 64-bit " - "consistent PCI DMA mask.\n"); - err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, " - "aborting.\n"); - goto err_free_res; - } - } - - mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev); - if (!mdev) { - dev_err(&pdev->dev, "Device struct alloc failed, " - "aborting.\n"); - err = -ENOMEM; - goto err_free_res; - } - - mdev->pdev = pdev; - - mdev->mthca_flags = mthca_hca_table[hca_type].flags; - if (ddr_hidden) - mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN; - - /* - * Now reset the HCA before we touch the PCI capabilities or - * attempt a firmware command, since a boot ROM may have left - * the HCA in an undefined state. - */ - err = mthca_reset(mdev); - if (err) { - mthca_err(mdev, "Failed to reset HCA, aborting.\n"); - goto err_free_dev; - } - - if (mthca_cmd_init(mdev)) { - mthca_err(mdev, "Failed to init command interface, aborting.\n"); - goto err_free_dev; - } - - err = mthca_tune_pci(mdev); - if (err) - goto err_cmd; - - err = mthca_init_hca(mdev); - if (err) - goto err_cmd; - - if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) { - mthca_warn(mdev, "HCA FW version %d.%d.%03d is old (%d.%d.%03d is current).\n", - (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff, - (int) (mdev->fw_ver & 0xffff), - (int) (mthca_hca_table[hca_type].latest_fw >> 32), - (int) (mthca_hca_table[hca_type].latest_fw >> 16) & 0xffff, - (int) (mthca_hca_table[hca_type].latest_fw & 0xffff)); - mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n"); - } - - if (msi_x && !mthca_enable_msi_x(mdev)) - mdev->mthca_flags |= MTHCA_FLAG_MSI_X; - - err = mthca_setup_hca(mdev); - if (err == -EBUSY && (mdev->mthca_flags & MTHCA_FLAG_MSI_X)) { - if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) - pci_disable_msix(pdev); - mdev->mthca_flags &= ~MTHCA_FLAG_MSI_X; - - err = mthca_setup_hca(mdev); - } - - if (err) - goto err_close; - - err = mthca_register_device(mdev); - if (err) - goto err_cleanup; - - err = mthca_create_agents(mdev); - if (err) - goto err_unregister; - - pci_set_drvdata(pdev, mdev); - mdev->hca_type = hca_type; - - mdev->active = 1; - - return 0; - -err_unregister: - mthca_unregister_device(mdev); - -err_cleanup: - mthca_cleanup_mcg_table(mdev); - mthca_cleanup_av_table(mdev); - mthca_cleanup_qp_table(mdev); - mthca_cleanup_srq_table(mdev); - mthca_cleanup_cq_table(mdev); - mthca_cmd_use_polling(mdev); - mthca_cleanup_eq_table(mdev); - - mthca_pd_free(mdev, &mdev->driver_pd); - - mthca_cleanup_mr_table(mdev); - mthca_cleanup_pd_table(mdev); - mthca_cleanup_uar_table(mdev); - -err_close: - if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) - pci_disable_msix(pdev); - - mthca_close_hca(mdev); - -err_cmd: - mthca_cmd_cleanup(mdev); - -err_free_dev: - ib_dealloc_device(&mdev->ib_dev); - -err_free_res: - pci_release_regions(pdev); - -err_disable_pdev: - pci_disable_device(pdev); - pci_set_drvdata(pdev, NULL); - return err; -} - -static void __mthca_remove_one(struct pci_dev *pdev) -{ - struct mthca_dev *mdev = pci_get_drvdata(pdev); - u8 status; - int p; - - if (mdev) { - mthca_free_agents(mdev); - mthca_unregister_device(mdev); - - for (p = 1; p <= mdev->limits.num_ports; ++p) - mthca_CLOSE_IB(mdev, p, &status); - - mthca_cleanup_mcg_table(mdev); - mthca_cleanup_av_table(mdev); - mthca_cleanup_qp_table(mdev); - mthca_cleanup_srq_table(mdev); - mthca_cleanup_cq_table(mdev); - mthca_cmd_use_polling(mdev); - mthca_cleanup_eq_table(mdev); - - mthca_pd_free(mdev, &mdev->driver_pd); - - mthca_cleanup_mr_table(mdev); - mthca_cleanup_pd_table(mdev); - - iounmap(mdev->kar); - mthca_uar_free(mdev, &mdev->driver_uar); - mthca_cleanup_uar_table(mdev); - mthca_close_hca(mdev); - mthca_cmd_cleanup(mdev); - - if (mdev->mthca_flags & MTHCA_FLAG_MSI_X) - pci_disable_msix(pdev); - - ib_dealloc_device(&mdev->ib_dev); - pci_release_regions(pdev); - pci_disable_device(pdev); - pci_set_drvdata(pdev, NULL); - } -} - -int __mthca_restart_one(struct pci_dev *pdev) -{ - struct mthca_dev *mdev; - int hca_type; - - mdev = pci_get_drvdata(pdev); - if (!mdev) - return -ENODEV; - hca_type = mdev->hca_type; - __mthca_remove_one(pdev); - return __mthca_init_one(pdev, hca_type); -} - -static int __devinit mthca_init_one(struct pci_dev *pdev, - const struct pci_device_id *id) -{ - static int mthca_version_printed = 0; - int ret; - - mutex_lock(&mthca_device_mutex); - - if (!mthca_version_printed) { - printk(KERN_INFO "%s", mthca_version); - ++mthca_version_printed; - } - - if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) { - printk(KERN_ERR PFX "%s has invalid driver data %jx\n", - pci_name(pdev), (uintmax_t)id->driver_data); - mutex_unlock(&mthca_device_mutex); - return -ENODEV; - } - - ret = __mthca_init_one(pdev, id->driver_data); - - mutex_unlock(&mthca_device_mutex); - - return ret; -} - -static void __devexit mthca_remove_one(struct pci_dev *pdev) -{ - mutex_lock(&mthca_device_mutex); - __mthca_remove_one(pdev); - mutex_unlock(&mthca_device_mutex); -} - -static struct pci_device_id mthca_pci_table[] = { - { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR), - .driver_data = TAVOR }, - { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR), - .driver_data = TAVOR }, - { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT), - .driver_data = ARBEL_COMPAT }, - { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT), - .driver_data = ARBEL_COMPAT }, - { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL), - .driver_data = ARBEL_NATIVE }, - { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL), - .driver_data = ARBEL_NATIVE }, - { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI), - .driver_data = SINAI }, - { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI), - .driver_data = SINAI }, - { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD), - .driver_data = SINAI }, - { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD), - .driver_data = SINAI }, - { 0, } -}; - -MODULE_DEVICE_TABLE(pci, mthca_pci_table); - -static struct pci_driver mthca_driver = { - .name = DRV_NAME, - .id_table = mthca_pci_table, - .probe = mthca_init_one, - .remove = __devexit_p(mthca_remove_one) -}; - -static void __init __mthca_check_profile_val(const char *name, int *pval, - int pval_default) -{ - /* value must be positive and power of 2 */ - int old_pval = *pval; - - if (old_pval <= 0) - *pval = pval_default; - else - *pval = roundup_pow_of_two(old_pval); - - if (old_pval != *pval) { - printk(KERN_WARNING PFX "Invalid value %d for %s in module parameter.\n", - old_pval, name); - printk(KERN_WARNING PFX "Corrected %s to %d.\n", name, *pval); - } -} - -#define mthca_check_profile_val(name, default) \ - __mthca_check_profile_val(#name, &hca_profile.name, default) - -static void __init mthca_validate_profile(void) -{ - mthca_check_profile_val(num_qp, MTHCA_DEFAULT_NUM_QP); - mthca_check_profile_val(rdb_per_qp, MTHCA_DEFAULT_RDB_PER_QP); - mthca_check_profile_val(num_cq, MTHCA_DEFAULT_NUM_CQ); - mthca_check_profile_val(num_mcg, MTHCA_DEFAULT_NUM_MCG); - mthca_check_profile_val(num_mpt, MTHCA_DEFAULT_NUM_MPT); - mthca_check_profile_val(num_mtt, MTHCA_DEFAULT_NUM_MTT); - mthca_check_profile_val(num_udav, MTHCA_DEFAULT_NUM_UDAV); - mthca_check_profile_val(fmr_reserved_mtts, MTHCA_DEFAULT_NUM_RESERVED_MTTS); - - if (hca_profile.fmr_reserved_mtts >= hca_profile.num_mtt) { - printk(KERN_WARNING PFX "Invalid fmr_reserved_mtts module parameter %d.\n", - hca_profile.fmr_reserved_mtts); - printk(KERN_WARNING PFX "(Must be smaller than num_mtt %d)\n", - hca_profile.num_mtt); - hca_profile.fmr_reserved_mtts = hca_profile.num_mtt / 2; - printk(KERN_WARNING PFX "Corrected fmr_reserved_mtts to %d.\n", - hca_profile.fmr_reserved_mtts); - } - if (log_mtts_per_seg == 0) - log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); - if ((log_mtts_per_seg < 1) || (log_mtts_per_seg > 5)) { - printk(KERN_WARNING PFX "bad log_mtts_per_seg (%d). Using default - %d\n", - log_mtts_per_seg, ilog2(MTHCA_MTT_SEG_SIZE / 8)); - log_mtts_per_seg = ilog2(MTHCA_MTT_SEG_SIZE / 8); - } -} - -static int __init mthca_init(void) -{ - int ret; - - mthca_validate_profile(); - - ret = mthca_catas_init(); - if (ret) - return ret; - - ret = pci_register_driver(&mthca_driver); - if (ret < 0) { - mthca_catas_cleanup(); - return ret; - } - - return 0; -} - -static void __exit mthca_cleanup(void) -{ - pci_unregister_driver(&mthca_driver); - mthca_catas_cleanup(); -} - -module_init_order(mthca_init, SI_ORDER_MIDDLE); -module_exit(mthca_cleanup); diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mcg.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_mcg.c deleted file mode 100644 index 6274a1c67aea..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mcg.c +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" - -struct mthca_mgm { - __be32 next_gid_index; - u32 reserved[3]; - u8 gid[16]; - __be32 qp[MTHCA_QP_PER_MGM]; -}; - -static const u8 zero_gid[16]; /* automatically initialized to 0 */ - -/* - * Caller must hold MCG table semaphore. gid and mgm parameters must - * be properly aligned for command interface. - * - * Returns 0 unless a firmware command error occurs. - * - * If GID is found in MGM or MGM is empty, *index = *hash, *prev = -1 - * and *mgm holds MGM entry. - * - * if GID is found in AMGM, *index = index in AMGM, *prev = index of - * previous entry in hash chain and *mgm holds AMGM entry. - * - * If no AMGM exists for given gid, *index = -1, *prev = index of last - * entry in hash chain and *mgm holds end of hash chain. - */ -static int find_mgm(struct mthca_dev *dev, - u8 *gid, struct mthca_mailbox *mgm_mailbox, - u16 *hash, int *prev, int *index) -{ - struct mthca_mailbox *mailbox; - struct mthca_mgm *mgm = mgm_mailbox->buf; - u8 *mgid; - int err; - u8 status; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return -ENOMEM; - mgid = mailbox->buf; - - memcpy(mgid, gid, 16); - - err = mthca_MGID_HASH(dev, mailbox, hash, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "MGID_HASH returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - if (0) - mthca_dbg(dev, "Hash for %pI6 is %04x\n", gid, *hash); - - *index = *hash; - *prev = -1; - - do { - err = mthca_READ_MGM(dev, *index, mgm_mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - if (!memcmp(mgm->gid, zero_gid, 16)) { - if (*index != *hash) { - mthca_err(dev, "Found zero MGID in AMGM.\n"); - err = -EINVAL; - } - goto out; - } - - if (!memcmp(mgm->gid, gid, 16)) - goto out; - - *prev = *index; - *index = be32_to_cpu(mgm->next_gid_index) >> 6; - } while (*index); - - *index = -1; - - out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_mailbox *mailbox; - struct mthca_mgm *mgm; - u16 hash; - int index, prev; - int link = 0; - int i; - int err; - u8 status; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - mgm = mailbox->buf; - - mutex_lock(&dev->mcg_table.mutex); - - err = find_mgm(dev, gid->raw, mailbox, &hash, &prev, &index); - if (err) - goto out; - - if (index != -1) { - if (!memcmp(mgm->gid, zero_gid, 16)) - memcpy(mgm->gid, gid->raw, 16); - } else { - link = 1; - - index = mthca_alloc(&dev->mcg_table.alloc); - if (index == -1) { - mthca_err(dev, "No AMGM entries left\n"); - err = -ENOMEM; - goto out; - } - - err = mthca_READ_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - memset(mgm, 0, sizeof *mgm); - memcpy(mgm->gid, gid->raw, 16); - } - - for (i = 0; i < MTHCA_QP_PER_MGM; ++i) - if (mgm->qp[i] == cpu_to_be32(ibqp->qp_num | (1U << 31))) { - mthca_dbg(dev, "QP %06x already a member of MGM\n", - ibqp->qp_num); - err = 0; - goto out; - } else if (!(mgm->qp[i] & cpu_to_be32(1U << 31))) { - mgm->qp[i] = cpu_to_be32(ibqp->qp_num | (1U << 31)); - break; - } - - if (i == MTHCA_QP_PER_MGM) { - mthca_err(dev, "MGM at index %x is full.\n", index); - err = -ENOMEM; - goto out; - } - - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - if (!link) - goto out; - - err = mthca_READ_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - mgm->next_gid_index = cpu_to_be32(index << 6); - - err = mthca_WRITE_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - } - - out: - if (err && link && index != -1) { - BUG_ON(index < dev->limits.num_mgms); - mthca_free(&dev->mcg_table.alloc, index); - } - mutex_unlock(&dev->mcg_table.mutex); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_mailbox *mailbox; - struct mthca_mgm *mgm; - u16 hash; - int prev, index; - int i, loc; - int err; - u8 status; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - mgm = mailbox->buf; - - mutex_lock(&dev->mcg_table.mutex); - - err = find_mgm(dev, gid->raw, mailbox, &hash, &prev, &index); - if (err) - goto out; - - if (index == -1) { - mthca_err(dev, "MGID %pI6 not found\n", gid->raw); - err = -EINVAL; - goto out; - } - - for (loc = -1, i = 0; i < MTHCA_QP_PER_MGM; ++i) { - if (mgm->qp[i] == cpu_to_be32(ibqp->qp_num | (1U << 31))) - loc = i; - if (!(mgm->qp[i] & cpu_to_be32(1U << 31))) - break; - } - - if (loc == -1) { - mthca_err(dev, "QP %06x not found in MGM\n", ibqp->qp_num); - err = -EINVAL; - goto out; - } - - mgm->qp[loc] = mgm->qp[i - 1]; - mgm->qp[i - 1] = 0; - - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - if (i != 1) - goto out; - - if (prev == -1) { - /* Remove entry from MGM */ - int amgm_index_to_free = be32_to_cpu(mgm->next_gid_index) >> 6; - if (amgm_index_to_free) { - err = mthca_READ_MGM(dev, amgm_index_to_free, - mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", - status); - err = -EINVAL; - goto out; - } - } else - memset(mgm->gid, 0, 16); - - err = mthca_WRITE_MGM(dev, index, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - if (amgm_index_to_free) { - BUG_ON(amgm_index_to_free < dev->limits.num_mgms); - mthca_free(&dev->mcg_table.alloc, amgm_index_to_free); - } - } else { - /* Remove entry from AMGM */ - int curr_next_index = be32_to_cpu(mgm->next_gid_index) >> 6; - err = mthca_READ_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "READ_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - - mgm->next_gid_index = cpu_to_be32(curr_next_index << 6); - - err = mthca_WRITE_MGM(dev, prev, mailbox, &status); - if (err) - goto out; - if (status) { - mthca_err(dev, "WRITE_MGM returned status %02x\n", status); - err = -EINVAL; - goto out; - } - BUG_ON(index < dev->limits.num_mgms); - mthca_free(&dev->mcg_table.alloc, index); - } - - out: - mutex_unlock(&dev->mcg_table.mutex); - - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_init_mcg_table(struct mthca_dev *dev) -{ - int err; - int table_size = dev->limits.num_mgms + dev->limits.num_amgms; - - err = mthca_alloc_init(&dev->mcg_table.alloc, - table_size, - table_size - 1, - dev->limits.num_mgms); - if (err) - return err; - - mutex_init(&dev->mcg_table.mutex); - - return 0; -} - -void mthca_cleanup_mcg_table(struct mthca_dev *dev) -{ - mthca_alloc_cleanup(&dev->mcg_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c deleted file mode 100644 index ad5818ee303a..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.c +++ /dev/null @@ -1,881 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include - -#include "mthca_memfree.h" -#include "mthca_dev.h" -#include "mthca_cmd.h" - -/* - * We allocate in as big chunks as we can, up to a maximum of 256 KB - * per chunk. - */ -enum { - MTHCA_ICM_ALLOC_SIZE = 1 << 18, - MTHCA_TABLE_CHUNK_SIZE = 1 << 18 -}; - -struct mthca_user_db_table { - struct mutex mutex; - struct { - u64 uvirt; - struct scatterlist mem; - int refcount; - } page[0]; -}; - -static void mthca_free_icm_pages(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) -{ - int i; - - if (chunk->nsg > 0) - pci_unmap_sg(dev->pdev, chunk->mem, chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - for (i = 0; i < chunk->npages; ++i) - __free_pages(sg_page(&chunk->mem[i]), - get_order(chunk->mem[i].length)); -} - -static void mthca_free_icm_coherent(struct mthca_dev *dev, struct mthca_icm_chunk *chunk) -{ - int i; - - for (i = 0; i < chunk->npages; ++i) { - dma_free_coherent(&dev->pdev->dev, chunk->mem[i].length, - lowmem_page_address(sg_page(&chunk->mem[i])), - sg_dma_address(&chunk->mem[i])); - } -} - -void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent) -{ - struct mthca_icm_chunk *chunk, *tmp; - - if (!icm) - return; - - list_for_each_entry_safe(chunk, tmp, &icm->chunk_list, list) { - if (coherent) - mthca_free_icm_coherent(dev, chunk); - else - mthca_free_icm_pages(dev, chunk); - - kfree(chunk); - } - - kfree(icm); -} - -static int mthca_alloc_icm_pages(struct scatterlist *mem, int order, gfp_t gfp_mask) -{ - struct page *page; - - /* - * Use __GFP_ZERO because buggy firmware assumes ICM pages are - * cleared, and subtle failures are seen if they aren't. - */ - page = alloc_pages(gfp_mask | __GFP_ZERO, order); - if (!page) - return -ENOMEM; - - sg_set_page(mem, page, PAGE_SIZE << order, 0); - return 0; -} - -static int mthca_alloc_icm_coherent(struct device *dev, struct scatterlist *mem, - int order, gfp_t gfp_mask) -{ - void *buf = dma_alloc_coherent(dev, PAGE_SIZE << order, &sg_dma_address(mem), - gfp_mask); - if (!buf) - return -ENOMEM; - - sg_set_buf(mem, buf, PAGE_SIZE << order); - BUG_ON(mem->offset); - sg_dma_len(mem) = PAGE_SIZE << order; - return 0; -} - -struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, - gfp_t gfp_mask, int coherent) -{ - struct mthca_icm *icm; - struct mthca_icm_chunk *chunk = NULL; - int cur_order; - int ret; - - /* We use sg_set_buf for coherent allocs, which assumes low memory */ - BUG_ON(coherent && (gfp_mask & __GFP_HIGHMEM)); - - icm = kmalloc(sizeof *icm, gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!icm) - return icm; - - icm->refcount = 0; - INIT_LIST_HEAD(&icm->chunk_list); - - cur_order = get_order(MTHCA_ICM_ALLOC_SIZE); - - while (npages > 0) { - if (!chunk) { - chunk = kmalloc(sizeof *chunk, - gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN)); - if (!chunk) - goto fail; - - sg_init_table(chunk->mem, MTHCA_ICM_CHUNK_LEN); - chunk->npages = 0; - chunk->nsg = 0; - list_add_tail(&chunk->list, &icm->chunk_list); - } - - while (1 << cur_order > npages) - --cur_order; - - if (coherent) - ret = mthca_alloc_icm_coherent(&dev->pdev->dev, - &chunk->mem[chunk->npages], - cur_order, gfp_mask); - else - ret = mthca_alloc_icm_pages(&chunk->mem[chunk->npages], - cur_order, gfp_mask); - - if (!ret) { - ++chunk->npages; - - if (coherent) - ++chunk->nsg; - else if (chunk->npages == MTHCA_ICM_CHUNK_LEN) { - chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, - chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - if (chunk->nsg <= 0) - goto fail; - } - - if (chunk->npages == MTHCA_ICM_CHUNK_LEN) - chunk = NULL; - - npages -= 1 << cur_order; - } else { - --cur_order; - if (cur_order < 0) - goto fail; - } - } - - if (!coherent && chunk) { - chunk->nsg = pci_map_sg(dev->pdev, chunk->mem, - chunk->npages, - PCI_DMA_BIDIRECTIONAL); - - if (chunk->nsg <= 0) - goto fail; - } - - return icm; - -fail: - mthca_free_icm(dev, icm, coherent); - return NULL; -} - -int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) -{ - int i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; - int ret = 0; - u8 status; - - mutex_lock(&table->mutex); - - if (table->icm[i]) { - ++table->icm[i]->refcount; - goto out; - } - - table->icm[i] = mthca_alloc_icm(dev, MTHCA_TABLE_CHUNK_SIZE >> PAGE_SHIFT, - (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN, table->coherent); - if (!table->icm[i]) { - ret = -ENOMEM; - goto out; - } - - if (mthca_MAP_ICM(dev, table->icm[i], table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - &status) || status) { - mthca_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; - ret = -ENOMEM; - goto out; - } - - ++table->icm[i]->refcount; - -out: - mutex_unlock(&table->mutex); - return ret; -} - -void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj) -{ - int i; - u8 status; - - if (!mthca_is_memfree(dev)) - return; - - i = (obj & (table->num_obj - 1)) * table->obj_size / MTHCA_TABLE_CHUNK_SIZE; - - mutex_lock(&table->mutex); - - if (--table->icm[i]->refcount == 0) { - mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; - } - - mutex_unlock(&table->mutex); -} - -void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle) -{ - int idx, offset, dma_offset, i; - struct mthca_icm_chunk *chunk; - struct mthca_icm *icm; - struct page *page = NULL; - - if (!table->lowmem) - return NULL; - - mutex_lock(&table->mutex); - - idx = (obj & (table->num_obj - 1)) * table->obj_size; - icm = table->icm[idx / MTHCA_TABLE_CHUNK_SIZE]; - dma_offset = offset = idx % MTHCA_TABLE_CHUNK_SIZE; - - if (!icm) - goto out; - - list_for_each_entry(chunk, &icm->chunk_list, list) { - for (i = 0; i < chunk->npages; ++i) { - if (dma_handle && dma_offset >= 0) { - if (sg_dma_len(&chunk->mem[i]) > dma_offset) - *dma_handle = sg_dma_address(&chunk->mem[i]) + - dma_offset; - dma_offset -= sg_dma_len(&chunk->mem[i]); - } - /* DMA mapping can merge pages but not split them, - * so if we found the page, dma_handle has already - * been assigned to. */ - if (chunk->mem[i].length > offset) { - page = sg_page(&chunk->mem[i]); - goto out; - } - offset -= chunk->mem[i].length; - } - } - -out: - mutex_unlock(&table->mutex); - return page ? lowmem_page_address(page) + offset : NULL; -} - -int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, - int start, int end) -{ - int inc = MTHCA_TABLE_CHUNK_SIZE / table->obj_size; - int i, err; - - for (i = start; i <= end; i += inc) { - err = mthca_table_get(dev, table, i); - if (err) - goto fail; - } - - return 0; - -fail: - while (i > start) { - i -= inc; - mthca_table_put(dev, table, i); - } - - return err; -} - -void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, - int start, int end) -{ - int i; - - if (!mthca_is_memfree(dev)) - return; - - for (i = start; i <= end; i += MTHCA_TABLE_CHUNK_SIZE / table->obj_size) - mthca_table_put(dev, table, i); -} - -struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, - u64 virt, int obj_size, - int nobj, int reserved, - int use_lowmem, int use_coherent) -{ - struct mthca_icm_table *table; - int obj_per_chunk; - int num_icm; - unsigned chunk_size; - int i; - u8 status; - - obj_per_chunk = MTHCA_TABLE_CHUNK_SIZE / obj_size; - num_icm = DIV_ROUND_UP(nobj, obj_per_chunk); - - table = kmalloc(sizeof *table + num_icm * sizeof *table->icm, GFP_KERNEL); - if (!table) - return NULL; - - table->virt = virt; - table->num_icm = num_icm; - table->num_obj = nobj; - table->obj_size = obj_size; - table->lowmem = use_lowmem; - table->coherent = use_coherent; - mutex_init(&table->mutex); - - for (i = 0; i < num_icm; ++i) - table->icm[i] = NULL; - - for (i = 0; i * MTHCA_TABLE_CHUNK_SIZE < reserved * obj_size; ++i) { - chunk_size = MTHCA_TABLE_CHUNK_SIZE; - if ((i + 1) * MTHCA_TABLE_CHUNK_SIZE > nobj * obj_size) - chunk_size = nobj * obj_size - i * MTHCA_TABLE_CHUNK_SIZE; - - table->icm[i] = mthca_alloc_icm(dev, chunk_size >> PAGE_SHIFT, - (use_lowmem ? GFP_KERNEL : GFP_HIGHUSER) | - __GFP_NOWARN, use_coherent); - if (!table->icm[i]) - goto err; - if (mthca_MAP_ICM(dev, table->icm[i], virt + i * MTHCA_TABLE_CHUNK_SIZE, - &status) || status) { - mthca_free_icm(dev, table->icm[i], table->coherent); - table->icm[i] = NULL; - goto err; - } - - /* - * Add a reference to this ICM chunk so that it never - * gets freed (since it contains reserved firmware objects). - */ - ++table->icm[i]->refcount; - } - - return table; - -err: - for (i = 0; i < num_icm; ++i) - if (table->icm[i]) { - mthca_UNMAP_ICM(dev, virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i], table->coherent); - } - - kfree(table); - - return NULL; -} - -void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table) -{ - int i; - u8 status; - - for (i = 0; i < table->num_icm; ++i) - if (table->icm[i]) { - mthca_UNMAP_ICM(dev, table->virt + i * MTHCA_TABLE_CHUNK_SIZE, - MTHCA_TABLE_CHUNK_SIZE / MTHCA_ICM_PAGE_SIZE, - &status); - mthca_free_icm(dev, table->icm[i], table->coherent); - } - - kfree(table); -} - -static u64 mthca_uarc_virt(struct mthca_dev *dev, struct mthca_uar *uar, int page) -{ - return dev->uar_table.uarc_base + - uar->index * dev->uar_table.uarc_size + - page * MTHCA_ICM_PAGE_SIZE; -} - - -#include -#include -#include - -#include -#include - -int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab, int index, u64 uaddr) -{ -#ifdef __linux__ - struct page *pages[1]; - int ret = 0; - u8 status; - int i; - - if (!mthca_is_memfree(dev)) - return 0; - - if (index < 0 || index > dev->uar_table.uarc_size / 8) - return -EINVAL; - - mutex_lock(&db_tab->mutex); - - i = index / MTHCA_DB_REC_PER_PAGE; - - if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || - (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || - (uaddr & 4095)) { - ret = -EINVAL; - goto out; - } - - if (db_tab->page[i].refcount) { - ++db_tab->page[i].refcount; - goto out; - } - - ret = get_user_pages(current, current->mm, uaddr & PAGE_MASK, 1, 1, 0, - pages, NULL); - if (ret < 0) - goto out; - - sg_set_page(&db_tab->page[i].mem, pages[0], MTHCA_ICM_PAGE_SIZE, - uaddr & ~PAGE_MASK); - - ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - if (ret < 0) { - put_page(pages[0]); - goto out; - } - - ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), - mthca_uarc_virt(dev, uar, i), &status); - if (!ret && status) - ret = -EINVAL; - if (ret) { - pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(sg_page(&db_tab->page[i].mem)); - goto out; - } - - db_tab->page[i].uvirt = uaddr; - db_tab->page[i].refcount = 1; - -out: - mutex_unlock(&db_tab->mutex); - return ret; -#else - struct proc *proc; - vm_offset_t start; - vm_paddr_t paddr; - pmap_t pmap; - vm_page_t m; - int ret = 0; - u8 status; - int i; - - if (!mthca_is_memfree(dev)) - return 0; - - if (index < 0 || index > dev->uar_table.uarc_size / 8) - return -EINVAL; - - mutex_lock(&db_tab->mutex); - - i = index / MTHCA_DB_REC_PER_PAGE; - start = 0; - - if ((db_tab->page[i].refcount >= MTHCA_DB_REC_PER_PAGE) || - (db_tab->page[i].uvirt && db_tab->page[i].uvirt != uaddr) || - (uaddr & 4095)) { - ret = -EINVAL; - goto out; - } - - if (db_tab->page[i].refcount) { - ++db_tab->page[i].refcount; - goto out; - } - - proc = curproc; - pmap = vm_map_pmap(&proc->p_vmspace->vm_map); - PROC_LOCK(proc); - if (ptoa(pmap_wired_count(pmap) + 1) > - lim_cur_proc(proc, RLIMIT_MEMLOCK)) { - PROC_UNLOCK(proc); - ret = -ENOMEM; - goto out; - } - PROC_UNLOCK(proc); - if (vm_cnt.v_wire_count + 1 > vm_page_max_wired) { - ret = -EAGAIN; - goto out; - } - start = uaddr & PAGE_MASK; - ret = vm_map_wire(&proc->p_vmspace->vm_map, start, start + PAGE_SIZE, - VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES | VM_MAP_WIRE_WRITE); - if (ret != KERN_SUCCESS) { - start = 0; - ret = -ENOMEM; - goto out; - } - paddr = pmap_extract(pmap, uaddr); - if (paddr == 0) { - ret = -EFAULT; - goto out; - } - m = PHYS_TO_VM_PAGE(paddr); - - sg_set_page(&db_tab->page[i].mem, m, MTHCA_ICM_PAGE_SIZE, - uaddr & ~PAGE_MASK); - - ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - if (ret < 0) - goto out; - - ret = mthca_MAP_ICM_page(dev, sg_dma_address(&db_tab->page[i].mem), - mthca_uarc_virt(dev, uar, i), &status); - if (!ret && status) - ret = -EINVAL; - if (ret) { - pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - goto out; - } - - db_tab->page[i].uvirt = uaddr; - db_tab->page[i].refcount = 1; - -out: - if (ret < 0 && start) - vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, - start, start + PAGE_SIZE, - VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); - mutex_unlock(&db_tab->mutex); - return ret; -#endif -} - -void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab, int index) -{ - if (!mthca_is_memfree(dev)) - return; - - /* - * To make our bookkeeping simpler, we don't unmap DB - * pages until we clean up the whole db table. - */ - - mutex_lock(&db_tab->mutex); - - --db_tab->page[index / MTHCA_DB_REC_PER_PAGE].refcount; - - mutex_unlock(&db_tab->mutex); -} - -struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev) -{ - struct mthca_user_db_table *db_tab; - int npages; - int i; - - if (!mthca_is_memfree(dev)) - return NULL; - - npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; - db_tab = kmalloc(sizeof *db_tab + npages * sizeof *db_tab->page, GFP_KERNEL); - if (!db_tab) - return ERR_PTR(-ENOMEM); - - mutex_init(&db_tab->mutex); - for (i = 0; i < npages; ++i) { - db_tab->page[i].refcount = 0; - db_tab->page[i].uvirt = 0; - sg_init_table(&db_tab->page[i].mem, 1); - } - - return db_tab; -} - -void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab) -{ - int i; - u8 status; - - if (!mthca_is_memfree(dev)) - return; - - for (i = 0; i < dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; ++i) { - if (db_tab->page[i].uvirt) { - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1, &status); - pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); -#ifdef __linux__ - put_page(sg_page(&db_tab->page[i].mem)); -#else - vm_offset_t start; - - start = db_tab->page[i].uvirt & PAGE_MASK; - vm_map_unwire(&curthread->td_proc->p_vmspace->vm_map, - start, start + PAGE_SIZE, - VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); -#endif - } - } - - kfree(db_tab); -} - -int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, - u32 qn, __be32 **db) -{ - int group; - int start, end, dir; - int i, j; - struct mthca_db_page *page; - int ret = 0; - u8 status; - - mutex_lock(&dev->db_tab->mutex); - - switch (type) { - case MTHCA_DB_TYPE_CQ_ARM: - case MTHCA_DB_TYPE_SQ: - group = 0; - start = 0; - end = dev->db_tab->max_group1; - dir = 1; - break; - - case MTHCA_DB_TYPE_CQ_SET_CI: - case MTHCA_DB_TYPE_RQ: - case MTHCA_DB_TYPE_SRQ: - group = 1; - start = dev->db_tab->npages - 1; - end = dev->db_tab->min_group2; - dir = -1; - break; - - default: - ret = -EINVAL; - goto out; - } - - for (i = start; i != end; i += dir) - if (dev->db_tab->page[i].db_rec && - !bitmap_full(dev->db_tab->page[i].used, - MTHCA_DB_REC_PER_PAGE)) { - page = dev->db_tab->page + i; - goto found; - } - - for (i = start; i != end; i += dir) - if (!dev->db_tab->page[i].db_rec) { - page = dev->db_tab->page + i; - goto alloc; - } - - if (dev->db_tab->max_group1 >= dev->db_tab->min_group2 - 1) { - ret = -ENOMEM; - goto out; - } - - if (group == 0) - ++dev->db_tab->max_group1; - else - --dev->db_tab->min_group2; - - page = dev->db_tab->page + end; - -alloc: - page->db_rec = dma_alloc_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, - &page->mapping, GFP_KERNEL); - if (!page->db_rec) { - ret = -ENOMEM; - goto out; - } - memset(page->db_rec, 0, MTHCA_ICM_PAGE_SIZE); - - ret = mthca_MAP_ICM_page(dev, page->mapping, - mthca_uarc_virt(dev, &dev->driver_uar, i), &status); - if (!ret && status) - ret = -EINVAL; - if (ret) { - dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, - page->db_rec, page->mapping); - goto out; - } - - bitmap_zero(page->used, MTHCA_DB_REC_PER_PAGE); - -found: - j = find_first_zero_bit(page->used, MTHCA_DB_REC_PER_PAGE); - set_bit(j, page->used); - - if (group == 1) - j = MTHCA_DB_REC_PER_PAGE - 1 - j; - - ret = i * MTHCA_DB_REC_PER_PAGE + j; - - page->db_rec[j] = cpu_to_be64((qn << 8) | (type << 5)); - - *db = (__be32 *) &page->db_rec[j]; - -out: - mutex_unlock(&dev->db_tab->mutex); - - return ret; -} - -void mthca_free_db(struct mthca_dev *dev, int type, int db_index) -{ - int i, j; - struct mthca_db_page *page; - u8 status; - - i = db_index / MTHCA_DB_REC_PER_PAGE; - j = db_index % MTHCA_DB_REC_PER_PAGE; - - page = dev->db_tab->page + i; - - mutex_lock(&dev->db_tab->mutex); - - page->db_rec[j] = 0; - if (i >= dev->db_tab->min_group2) - j = MTHCA_DB_REC_PER_PAGE - 1 - j; - clear_bit(j, page->used); - - if (bitmap_empty(page->used, MTHCA_DB_REC_PER_PAGE) && - i >= dev->db_tab->max_group1 - 1) { - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); - - dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, - page->db_rec, page->mapping); - page->db_rec = NULL; - - if (i == dev->db_tab->max_group1) { - --dev->db_tab->max_group1; - /* XXX may be able to unmap more pages now */ - } - if (i == dev->db_tab->min_group2) - ++dev->db_tab->min_group2; - } - - mutex_unlock(&dev->db_tab->mutex); -} - -int mthca_init_db_tab(struct mthca_dev *dev) -{ - int i; - - if (!mthca_is_memfree(dev)) - return 0; - - dev->db_tab = kmalloc(sizeof *dev->db_tab, GFP_KERNEL); - if (!dev->db_tab) - return -ENOMEM; - - mutex_init(&dev->db_tab->mutex); - - dev->db_tab->npages = dev->uar_table.uarc_size / MTHCA_ICM_PAGE_SIZE; - dev->db_tab->max_group1 = 0; - dev->db_tab->min_group2 = dev->db_tab->npages - 1; - - dev->db_tab->page = kmalloc(dev->db_tab->npages * - sizeof *dev->db_tab->page, - GFP_KERNEL); - if (!dev->db_tab->page) { - kfree(dev->db_tab); - return -ENOMEM; - } - - for (i = 0; i < dev->db_tab->npages; ++i) - dev->db_tab->page[i].db_rec = NULL; - - return 0; -} - -void mthca_cleanup_db_tab(struct mthca_dev *dev) -{ - int i; - u8 status; - - if (!mthca_is_memfree(dev)) - return; - - /* - * Because we don't always free our UARC pages when they - * become empty to make mthca_free_db() simpler we need to - * make a sweep through the doorbell pages and free any - * leftover pages now. - */ - for (i = 0; i < dev->db_tab->npages; ++i) { - if (!dev->db_tab->page[i].db_rec) - continue; - - if (!bitmap_empty(dev->db_tab->page[i].used, MTHCA_DB_REC_PER_PAGE)) - mthca_warn(dev, "Kernel UARC page %d not empty\n", i); - - mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, &dev->driver_uar, i), 1, &status); - - dma_free_coherent(&dev->pdev->dev, MTHCA_ICM_PAGE_SIZE, - dev->db_tab->page[i].db_rec, - dev->db_tab->page[i].mapping); - } - - kfree(dev->db_tab->page); - kfree(dev->db_tab); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.h deleted file mode 100644 index da9b8f9b884f..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_memfree.h +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_MEMFREE_H -#define MTHCA_MEMFREE_H - -#include -#include - -#define MTHCA_ICM_CHUNK_LEN \ - ((256 - sizeof (struct list_head) - 2 * sizeof (int)) / \ - (sizeof (struct scatterlist))) - -enum { - MTHCA_ICM_PAGE_SHIFT = 12, - MTHCA_ICM_PAGE_SIZE = 1 << MTHCA_ICM_PAGE_SHIFT, - MTHCA_DB_REC_PER_PAGE = MTHCA_ICM_PAGE_SIZE / 8 -}; - -struct mthca_icm_chunk { - struct list_head list; - int npages; - int nsg; - struct scatterlist mem[MTHCA_ICM_CHUNK_LEN]; -}; - -struct mthca_icm { - struct list_head chunk_list; - int refcount; -}; - -struct mthca_icm_table { - u64 virt; - int num_icm; - int num_obj; - int obj_size; - int lowmem; - int coherent; - struct mutex mutex; - struct mthca_icm *icm[0]; -}; - -struct mthca_icm_iter { - struct mthca_icm *icm; - struct mthca_icm_chunk *chunk; - int page_idx; -}; - -struct mthca_dev; - -struct mthca_icm *mthca_alloc_icm(struct mthca_dev *dev, int npages, - gfp_t gfp_mask, int coherent); -void mthca_free_icm(struct mthca_dev *dev, struct mthca_icm *icm, int coherent); - -struct mthca_icm_table *mthca_alloc_icm_table(struct mthca_dev *dev, - u64 virt, int obj_size, - int nobj, int reserved, - int use_lowmem, int use_coherent); -void mthca_free_icm_table(struct mthca_dev *dev, struct mthca_icm_table *table); -int mthca_table_get(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); -void mthca_table_put(struct mthca_dev *dev, struct mthca_icm_table *table, int obj); -void *mthca_table_find(struct mthca_icm_table *table, int obj, dma_addr_t *dma_handle); -int mthca_table_get_range(struct mthca_dev *dev, struct mthca_icm_table *table, - int start, int end); -void mthca_table_put_range(struct mthca_dev *dev, struct mthca_icm_table *table, - int start, int end); - -static inline void mthca_icm_first(struct mthca_icm *icm, - struct mthca_icm_iter *iter) -{ - iter->icm = icm; - iter->chunk = list_empty(&icm->chunk_list) ? - NULL : list_entry(icm->chunk_list.next, - struct mthca_icm_chunk, list); - iter->page_idx = 0; -} - -static inline int mthca_icm_last(struct mthca_icm_iter *iter) -{ - return !iter->chunk; -} - -static inline void mthca_icm_next(struct mthca_icm_iter *iter) -{ - if (++iter->page_idx >= iter->chunk->nsg) { - if (iter->chunk->list.next == &iter->icm->chunk_list) { - iter->chunk = NULL; - return; - } - - iter->chunk = list_entry(iter->chunk->list.next, - struct mthca_icm_chunk, list); - iter->page_idx = 0; - } -} - -static inline dma_addr_t mthca_icm_addr(struct mthca_icm_iter *iter) -{ - return sg_dma_address(&iter->chunk->mem[iter->page_idx]); -} - -static inline unsigned long mthca_icm_size(struct mthca_icm_iter *iter) -{ - return sg_dma_len(&iter->chunk->mem[iter->page_idx]); -} - -struct mthca_db_page { - DECLARE_BITMAP(used, MTHCA_DB_REC_PER_PAGE); - __be64 *db_rec; - dma_addr_t mapping; -}; - -struct mthca_db_table { - int npages; - int max_group1; - int min_group2; - struct mthca_db_page *page; - struct mutex mutex; -}; - -enum mthca_db_type { - MTHCA_DB_TYPE_INVALID = 0x0, - MTHCA_DB_TYPE_CQ_SET_CI = 0x1, - MTHCA_DB_TYPE_CQ_ARM = 0x2, - MTHCA_DB_TYPE_SQ = 0x3, - MTHCA_DB_TYPE_RQ = 0x4, - MTHCA_DB_TYPE_SRQ = 0x5, - MTHCA_DB_TYPE_GROUP_SEP = 0x7 -}; - -struct mthca_user_db_table; -struct mthca_uar; - -int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab, int index, u64 uaddr); -void mthca_unmap_user_db(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab, int index); -struct mthca_user_db_table *mthca_init_user_db_tab(struct mthca_dev *dev); -void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, - struct mthca_user_db_table *db_tab); - -int mthca_init_db_tab(struct mthca_dev *dev); -void mthca_cleanup_db_tab(struct mthca_dev *dev); -int mthca_alloc_db(struct mthca_dev *dev, enum mthca_db_type type, - u32 qn, __be32 **db); -void mthca_free_db(struct mthca_dev *dev, int type, int db_index); - -#endif /* MTHCA_MEMFREE_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c deleted file mode 100644 index d606edf10858..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_mr.c +++ /dev/null @@ -1,985 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_memfree.h" - -struct mthca_mtt { - struct mthca_buddy *buddy; - int order; - u32 first_seg; -}; - -/* - * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. - */ -struct mthca_mpt_entry { - __be32 flags; - __be32 page_size; - __be32 key; - __be32 pd; - __be64 start; - __be64 length; - __be32 lkey; - __be32 window_count; - __be32 window_count_limit; - __be64 mtt_seg; - __be32 mtt_sz; /* Arbel only */ - u32 reserved[2]; -} __attribute__((packed)); - -#define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) -#define MTHCA_MPT_FLAG_MIO (1 << 17) -#define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) -#define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) -#define MTHCA_MPT_FLAG_REGION (1 << 8) - -#define MTHCA_MTT_FLAG_PRESENT 1 - -#define MTHCA_MPT_STATUS_SW 0xF0 -#define MTHCA_MPT_STATUS_HW 0x00 - -#define SINAI_FMR_KEY_INC 0x1000000 - -/* - * Buddy allocator for MTT segments (currently not very efficient - * since it doesn't keep a free list and just searches linearly - * through the bitmaps) - */ - -static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) -{ - int o; - int m; - u32 seg; - - spin_lock(&buddy->lock); - - for (o = order; o <= buddy->max_order; ++o) - if (buddy->num_free[o]) { - m = 1 << (buddy->max_order - o); - seg = find_first_bit(buddy->bits[o], m); - if (seg < m) - goto found; - } - - spin_unlock(&buddy->lock); - return -1; - - found: - clear_bit(seg, buddy->bits[o]); - --buddy->num_free[o]; - - while (o > order) { - --o; - seg <<= 1; - set_bit(seg ^ 1, buddy->bits[o]); - ++buddy->num_free[o]; - } - - spin_unlock(&buddy->lock); - - seg <<= order; - - return seg; -} - -static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) -{ - seg >>= order; - - spin_lock(&buddy->lock); - - while (test_bit(seg ^ 1, buddy->bits[order])) { - clear_bit(seg ^ 1, buddy->bits[order]); - --buddy->num_free[order]; - seg >>= 1; - ++order; - } - - set_bit(seg, buddy->bits[order]); - ++buddy->num_free[order]; - - spin_unlock(&buddy->lock); -} - -static int mthca_buddy_init(struct mthca_buddy *buddy, int max_order) -{ - int i, s; - - buddy->max_order = max_order; - spin_lock_init(&buddy->lock); - - buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), - GFP_KERNEL); - buddy->num_free = kzalloc((buddy->max_order + 1) * sizeof (int *), - GFP_KERNEL); - if (!buddy->bits || !buddy->num_free) - goto err_out; - - for (i = 0; i <= buddy->max_order; ++i) { - s = BITS_TO_LONGS(1 << (buddy->max_order - i)); - buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); - if (!buddy->bits[i]) - goto err_out_free; - bitmap_zero(buddy->bits[i], - 1 << (buddy->max_order - i)); - } - - set_bit(0, buddy->bits[buddy->max_order]); - buddy->num_free[buddy->max_order] = 1; - - return 0; - -err_out_free: - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); - -err_out: - kfree(buddy->bits); - kfree(buddy->num_free); - - return -ENOMEM; -} - -static void mthca_buddy_cleanup(struct mthca_buddy *buddy) -{ - int i; - - for (i = 0; i <= buddy->max_order; ++i) - kfree(buddy->bits[i]); - - kfree(buddy->bits); - kfree(buddy->num_free); -} - -static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, - struct mthca_buddy *buddy) -{ - u32 seg = mthca_buddy_alloc(buddy, order); - - if (seg == -1) - return -1; - - if (mthca_is_memfree(dev)) - if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, - seg + (1 << order) - 1)) { - mthca_buddy_free(buddy, seg, order); - seg = -1; - } - - return seg; -} - -static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, - struct mthca_buddy *buddy) -{ - struct mthca_mtt *mtt; - int i; - - if (size <= 0) - return ERR_PTR(-EINVAL); - - mtt = kmalloc(sizeof *mtt, GFP_KERNEL); - if (!mtt) - return ERR_PTR(-ENOMEM); - - mtt->buddy = buddy; - mtt->order = 0; - for (i = dev->limits.mtt_seg_size / 8; i < size; i <<= 1) - ++mtt->order; - - mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); - if (mtt->first_seg == -1) { - kfree(mtt); - return ERR_PTR(-ENOMEM); - } - - return mtt; -} - -struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size) -{ - return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy); -} - -void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) -{ - if (!mtt) - return; - - mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order); - - mthca_table_put_range(dev, dev->mr_table.mtt_table, - mtt->first_seg, - mtt->first_seg + (1 << mtt->order) - 1); - - kfree(mtt); -} - -static int __mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, - int start_index, u64 *buffer_list, int list_len) -{ - struct mthca_mailbox *mailbox; - __be64 *mtt_entry; - int err = 0; - u8 status; - int i; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - mtt_entry = mailbox->buf; - - while (list_len > 0) { - mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + - mtt->first_seg * dev->limits.mtt_seg_size + - start_index * 8); - mtt_entry[1] = 0; - for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) - mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | - MTHCA_MTT_FLAG_PRESENT); - - /* - * If we have an odd number of entries to write, add - * one more dummy entry for firmware efficiency. - */ - if (i & 1) - mtt_entry[i + 2] = 0; - - err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); - if (err) { - mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); - goto out; - } - if (status) { - mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", - status); - err = -EINVAL; - goto out; - } - - list_len -= i; - start_index += i; - buffer_list += i; - } - -out: - mthca_free_mailbox(dev, mailbox); - return err; -} - -int mthca_write_mtt_size(struct mthca_dev *dev) -{ - if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || - !(dev->mthca_flags & MTHCA_FLAG_FMR)) - /* - * Be friendly to WRITE_MTT command - * and leave two empty slots for the - * index and reserved fields of the - * mailbox. - */ - return PAGE_SIZE / sizeof (u64) - 2; - - /* For Arbel, all MTTs must fit in the same page. */ - return mthca_is_memfree(dev) ? (PAGE_SIZE / sizeof (u64)) : 0x7ffffff; -} - -static void mthca_tavor_write_mtt_seg(struct mthca_dev *dev, - struct mthca_mtt *mtt, int start_index, - u64 *buffer_list, int list_len) -{ - u64 __iomem *mtts; - int i; - - mtts = dev->mr_table.tavor_fmr.mtt_base + mtt->first_seg * dev->limits.mtt_seg_size + - start_index * sizeof (u64); - for (i = 0; i < list_len; ++i) - mthca_write64_raw(cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT), - mtts + i); -} - -static void mthca_arbel_write_mtt_seg(struct mthca_dev *dev, - struct mthca_mtt *mtt, int start_index, - u64 *buffer_list, int list_len) -{ - __be64 *mtts; - dma_addr_t dma_handle; - int i; - int s = start_index * sizeof (u64); - - /* For Arbel, all MTTs must fit in the same page. */ - BUG_ON(s / PAGE_SIZE != (s + list_len * sizeof(u64) - 1) / PAGE_SIZE); - /* Require full segments */ - BUG_ON(s % dev->limits.mtt_seg_size); - - mtts = mthca_table_find(dev->mr_table.mtt_table, mtt->first_seg + - s / dev->limits.mtt_seg_size, &dma_handle); - - BUG_ON(!mtts); - - for (i = 0; i < list_len; ++i) - mtts[i] = cpu_to_be64(buffer_list[i] | MTHCA_MTT_FLAG_PRESENT); - - dma_sync_single(&dev->pdev->dev, dma_handle, list_len * sizeof (u64), DMA_TO_DEVICE); -} - -int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, - int start_index, u64 *buffer_list, int list_len) -{ - int size = mthca_write_mtt_size(dev); - int chunk; - - if (dev->mr_table.fmr_mtt_buddy != &dev->mr_table.mtt_buddy || - !(dev->mthca_flags & MTHCA_FLAG_FMR)) - return __mthca_write_mtt(dev, mtt, start_index, buffer_list, list_len); - - while (list_len > 0) { - chunk = min(size, list_len); - if (mthca_is_memfree(dev)) - mthca_arbel_write_mtt_seg(dev, mtt, start_index, - buffer_list, chunk); - else - mthca_tavor_write_mtt_seg(dev, mtt, start_index, - buffer_list, chunk); - - list_len -= chunk; - start_index += chunk; - buffer_list += chunk; - } - - return 0; -} - -static inline u32 tavor_hw_index_to_key(u32 ind) -{ - return ind; -} - -static inline u32 tavor_key_to_hw_index(u32 key) -{ - return key; -} - -static inline u32 arbel_hw_index_to_key(u32 ind) -{ - return (ind >> 24) | (ind << 8); -} - -static inline u32 arbel_key_to_hw_index(u32 key) -{ - return (key << 24) | (key >> 8); -} - -static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) -{ - if (mthca_is_memfree(dev)) - return arbel_hw_index_to_key(ind); - else - return tavor_hw_index_to_key(ind); -} - -static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) -{ - if (mthca_is_memfree(dev)) - return arbel_key_to_hw_index(key); - else - return tavor_key_to_hw_index(key); -} - -static inline u32 adjust_key(struct mthca_dev *dev, u32 key) -{ - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - return ((key << 20) & 0x800000) | (key & 0x7fffff); - else - return key; -} - -int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, - u64 iova, u64 total_size, u32 access, struct mthca_mr *mr) -{ - struct mthca_mailbox *mailbox; - struct mthca_mpt_entry *mpt_entry; - u32 key; - int i; - int err; - u8 status; - - WARN_ON(buffer_size_shift >= 32); - - key = mthca_alloc(&dev->mr_table.mpt_alloc); - if (key == -1) - return -ENOMEM; - key = adjust_key(dev, key); - mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); - - if (mthca_is_memfree(dev)) { - err = mthca_table_get(dev, dev->mr_table.mpt_table, key); - if (err) - goto err_out_mpt_free; - } - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_out_table; - } - mpt_entry = mailbox->buf; - - mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | - MTHCA_MPT_FLAG_MIO | - MTHCA_MPT_FLAG_REGION | - access); - if (!mr->mtt) - mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL); - - mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); - mpt_entry->key = cpu_to_be32(key); - mpt_entry->pd = cpu_to_be32(pd); - mpt_entry->start = cpu_to_be64(iova); - mpt_entry->length = cpu_to_be64(total_size); - - memset(&mpt_entry->lkey, 0, - sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); - - if (mr->mtt) - mpt_entry->mtt_seg = - cpu_to_be64(dev->mr_table.mtt_base + - mr->mtt->first_seg * dev->limits.mtt_seg_size); - - if (0) { - mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); - for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { - if (i % 4 == 0) - printk("[%02x] ", i * 4); - printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); - if ((i + 1) % 4 == 0) - printk("\n"); - } - } - - err = mthca_SW2HW_MPT(dev, mailbox, - key & (dev->limits.num_mpts - 1), - &status); - if (err) { - mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); - goto err_out_mailbox; - } else if (status) { - mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_mailbox; - } - - mthca_free_mailbox(dev, mailbox); - return err; - -err_out_mailbox: - mthca_free_mailbox(dev, mailbox); - -err_out_table: - mthca_table_put(dev, dev->mr_table.mpt_table, key); - -err_out_mpt_free: - mthca_free(&dev->mr_table.mpt_alloc, key); - return err; -} - -int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, - u32 access, struct mthca_mr *mr) -{ - mr->mtt = NULL; - return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr); -} - -int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, - u64 *buffer_list, int buffer_size_shift, - int list_len, u64 iova, u64 total_size, - u32 access, struct mthca_mr *mr) -{ - int err; - - mr->mtt = mthca_alloc_mtt(dev, list_len); - if (IS_ERR(mr->mtt)) - return PTR_ERR(mr->mtt); - - err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len); - if (err) { - mthca_free_mtt(dev, mr->mtt); - return err; - } - - err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova, - total_size, access, mr); - if (err) - mthca_free_mtt(dev, mr->mtt); - - return err; -} - -/* Free mr or fmr */ -static void mthca_free_region(struct mthca_dev *dev, u32 lkey) -{ - mthca_table_put(dev, dev->mr_table.mpt_table, - key_to_hw_index(dev, lkey)); - - mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey)); -} - -void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) -{ - int err; - u8 status; - - err = mthca_HW2SW_MPT(dev, NULL, - key_to_hw_index(dev, mr->ibmr.lkey) & - (dev->limits.num_mpts - 1), - &status); - if (err) - mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", - status); - - mthca_free_region(dev, mr->ibmr.lkey); - mthca_free_mtt(dev, mr->mtt); -} - -int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, - u32 access, struct mthca_fmr *mr) -{ - struct mthca_mpt_entry *mpt_entry; - struct mthca_mailbox *mailbox; - u64 mtt_seg; - u32 key, idx; - u8 status; - int list_len = mr->attr.max_pages; - int err = -ENOMEM; - int i; - - if (mr->attr.page_shift < 12 || mr->attr.page_shift >= 32) - return -EINVAL; - - /* For Arbel, all MTTs must fit in the same page. */ - if (mthca_is_memfree(dev) && - mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) - return -EINVAL; - - mr->maps = 0; - - key = mthca_alloc(&dev->mr_table.mpt_alloc); - if (key == -1) - return -ENOMEM; - key = adjust_key(dev, key); - - idx = key & (dev->limits.num_mpts - 1); - mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); - - if (mthca_is_memfree(dev)) { - err = mthca_table_get(dev, dev->mr_table.mpt_table, key); - if (err) - goto err_out_mpt_free; - - mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key, NULL); - BUG_ON(!mr->mem.arbel.mpt); - } else - mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + - sizeof *(mr->mem.tavor.mpt) * idx; - - mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); - if (IS_ERR(mr->mtt)) { - err = PTR_ERR(mr->mtt); - goto err_out_table; - } - - mtt_seg = mr->mtt->first_seg * dev->limits.mtt_seg_size; - - if (mthca_is_memfree(dev)) { - mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, - mr->mtt->first_seg, - &mr->mem.arbel.dma_handle); - BUG_ON(!mr->mem.arbel.mtts); - } else - mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_out_free_mtt; - } - - mpt_entry = mailbox->buf; - - mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | - MTHCA_MPT_FLAG_MIO | - MTHCA_MPT_FLAG_REGION | - access); - - mpt_entry->page_size = cpu_to_be32(mr->attr.page_shift - 12); - mpt_entry->key = cpu_to_be32(key); - mpt_entry->pd = cpu_to_be32(pd); - memset(&mpt_entry->start, 0, - sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); - mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); - - if (0) { - mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); - for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { - if (i % 4 == 0) - printk("[%02x] ", i * 4); - printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); - if ((i + 1) % 4 == 0) - printk("\n"); - } - } - - err = mthca_SW2HW_MPT(dev, mailbox, - key & (dev->limits.num_mpts - 1), - &status); - if (err) { - mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); - goto err_out_mailbox_free; - } - if (status) { - mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_mailbox_free; - } - - mthca_free_mailbox(dev, mailbox); - return 0; - -err_out_mailbox_free: - mthca_free_mailbox(dev, mailbox); - -err_out_free_mtt: - mthca_free_mtt(dev, mr->mtt); - -err_out_table: - mthca_table_put(dev, dev->mr_table.mpt_table, key); - -err_out_mpt_free: - mthca_free(&dev->mr_table.mpt_alloc, key); - return err; -} - -int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) -{ - if (fmr->maps) - return -EBUSY; - - mthca_free_region(dev, fmr->ibmr.lkey); - mthca_free_mtt(dev, fmr->mtt); - - return 0; -} - -static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, - int list_len, u64 iova) -{ - int i, page_mask; - - if (list_len > fmr->attr.max_pages) - return -EINVAL; - - page_mask = (1 << fmr->attr.page_shift) - 1; - - /* We are getting page lists, so va must be page aligned. */ - if (iova & page_mask) - return -EINVAL; - - /* Trust the user not to pass misaligned data in page_list */ - if (0) - for (i = 0; i < list_len; ++i) { - if (page_list[i] & ~page_mask) - return -EINVAL; - } - - if (fmr->maps >= fmr->attr.max_maps) - return -EINVAL; - - return 0; -} - - -int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, - int list_len, u64 iova) -{ - struct mthca_fmr *fmr = to_mfmr(ibfmr); - struct mthca_dev *dev = to_mdev(ibfmr->device); - struct mthca_mpt_entry mpt_entry; - u32 key; - int i, err; - - err = mthca_check_fmr(fmr, page_list, list_len, iova); - if (err) - return err; - - ++fmr->maps; - - key = tavor_key_to_hw_index(fmr->ibmr.lkey); - key += dev->limits.num_mpts; - fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); - - writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); - - for (i = 0; i < list_len; ++i) { - __be64 mtt_entry = cpu_to_be64(page_list[i] | - MTHCA_MTT_FLAG_PRESENT); - mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); - } - - mpt_entry.lkey = cpu_to_be32(key); - mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); - mpt_entry.start = cpu_to_be64(iova); - - __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); - memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, - offsetof(struct mthca_mpt_entry, window_count) - - offsetof(struct mthca_mpt_entry, start)); - - writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); - - return 0; -} - -int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, - int list_len, u64 iova) -{ - struct mthca_fmr *fmr = to_mfmr(ibfmr); - struct mthca_dev *dev = to_mdev(ibfmr->device); - u32 key; - int i, err; - - err = mthca_check_fmr(fmr, page_list, list_len, iova); - if (err) - return err; - - ++fmr->maps; - - key = arbel_key_to_hw_index(fmr->ibmr.lkey); - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - key += SINAI_FMR_KEY_INC; - else - key += dev->limits.num_mpts; - fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); - - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; - - wmb(); - - for (i = 0; i < list_len; ++i) - fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | - MTHCA_MTT_FLAG_PRESENT); - - dma_sync_single(&dev->pdev->dev, fmr->mem.arbel.dma_handle, - list_len * sizeof(u64), DMA_TO_DEVICE); - - fmr->mem.arbel.mpt->key = cpu_to_be32(key); - fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); - fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_shift)); - fmr->mem.arbel.mpt->start = cpu_to_be64(iova); - - wmb(); - - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; - - wmb(); - - return 0; -} - -void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) -{ - if (!fmr->maps) - return; - - fmr->maps = 0; - - writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); -} - -void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) -{ - if (!fmr->maps) - return; - - fmr->maps = 0; - - *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; -} - -int mthca_init_mr_table(struct mthca_dev *dev) -{ - unsigned long addr; - int mpts, mtts, err, i; - - err = mthca_alloc_init(&dev->mr_table.mpt_alloc, - dev->limits.num_mpts, - ~0, dev->limits.reserved_mrws); - if (err) - return err; - - if (!mthca_is_memfree(dev) && - (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) - dev->limits.fmr_reserved_mtts = 0; - else - dev->mthca_flags |= MTHCA_FLAG_FMR; - - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - mthca_dbg(dev, "Memory key throughput optimization activated.\n"); - - err = mthca_buddy_init(&dev->mr_table.mtt_buddy, - fls(dev->limits.num_mtt_segs - 1)); - - if (err) - goto err_mtt_buddy; - - dev->mr_table.tavor_fmr.mpt_base = NULL; - dev->mr_table.tavor_fmr.mtt_base = NULL; - - if (dev->limits.fmr_reserved_mtts) { - i = fls(dev->limits.fmr_reserved_mtts - 1); - - if (i >= 31) { - mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n"); - err = -EINVAL; - goto err_fmr_mpt; - } - mpts = mtts = 1 << i; - } else { - mtts = dev->limits.num_mtt_segs; - mpts = dev->limits.num_mpts; - } - - if (!mthca_is_memfree(dev) && - (dev->mthca_flags & MTHCA_FLAG_FMR)) { - - addr = pci_resource_start(dev->pdev, 4) + - ((pci_resource_len(dev->pdev, 4) - 1) & - dev->mr_table.mpt_base); - - dev->mr_table.tavor_fmr.mpt_base = - ioremap(addr, mpts * sizeof(struct mthca_mpt_entry)); - - if (!dev->mr_table.tavor_fmr.mpt_base) { - mthca_warn(dev, "MPT ioremap for FMR failed.\n"); - err = -ENOMEM; - goto err_fmr_mpt; - } - - addr = pci_resource_start(dev->pdev, 4) + - ((pci_resource_len(dev->pdev, 4) - 1) & - dev->mr_table.mtt_base); - - dev->mr_table.tavor_fmr.mtt_base = - ioremap(addr, mtts * dev->limits.mtt_seg_size); - if (!dev->mr_table.tavor_fmr.mtt_base) { - mthca_warn(dev, "MTT ioremap for FMR failed.\n"); - err = -ENOMEM; - goto err_fmr_mtt; - } - } - - if (dev->limits.fmr_reserved_mtts) { - err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, fls(mtts - 1)); - if (err) - goto err_fmr_mtt_buddy; - - /* Prevent regular MRs from using FMR keys */ - err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, fls(mtts - 1)); - if (err) - goto err_reserve_fmr; - - dev->mr_table.fmr_mtt_buddy = - &dev->mr_table.tavor_fmr.mtt_buddy; - } else - dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy; - - /* FMR table is always the first, take reserved MTTs out of there */ - if (dev->limits.reserved_mtts) { - i = fls(dev->limits.reserved_mtts - 1); - - if (mthca_alloc_mtt_range(dev, i, - dev->mr_table.fmr_mtt_buddy) == -1) { - mthca_warn(dev, "MTT table of order %d is too small.\n", - dev->mr_table.fmr_mtt_buddy->max_order); - err = -ENOMEM; - goto err_reserve_mtts; - } - } - - return 0; - -err_reserve_mtts: -err_reserve_fmr: - if (dev->limits.fmr_reserved_mtts) - mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); - -err_fmr_mtt_buddy: - if (dev->mr_table.tavor_fmr.mtt_base) - iounmap(dev->mr_table.tavor_fmr.mtt_base); - -err_fmr_mtt: - if (dev->mr_table.tavor_fmr.mpt_base) - iounmap(dev->mr_table.tavor_fmr.mpt_base); - -err_fmr_mpt: - mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); - -err_mtt_buddy: - mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); - - return err; -} - -void mthca_cleanup_mr_table(struct mthca_dev *dev) -{ - /* XXX check if any MRs are still allocated? */ - if (dev->limits.fmr_reserved_mtts) - mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); - - mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); - - if (dev->mr_table.tavor_fmr.mtt_base) - iounmap(dev->mr_table.tavor_fmr.mtt_base); - if (dev->mr_table.tavor_fmr.mpt_base) - iounmap(dev->mr_table.tavor_fmr.mpt_base); - - mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_pd.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_pd.c deleted file mode 100644 index 266f14e47406..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_pd.c +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#include "mthca_dev.h" - -int mthca_pd_alloc(struct mthca_dev *dev, int privileged, struct mthca_pd *pd) -{ - int err = 0; - - pd->privileged = privileged; - - atomic_set(&pd->sqp_count, 0); - pd->pd_num = mthca_alloc(&dev->pd_table.alloc); - if (pd->pd_num == -1) - return -ENOMEM; - - if (privileged) { - err = mthca_mr_alloc_notrans(dev, pd->pd_num, - MTHCA_MPT_FLAG_LOCAL_READ | - MTHCA_MPT_FLAG_LOCAL_WRITE, - &pd->ntmr); - if (err) - mthca_free(&dev->pd_table.alloc, pd->pd_num); - } - - return err; -} - -void mthca_pd_free(struct mthca_dev *dev, struct mthca_pd *pd) -{ - if (pd->privileged) - mthca_free_mr(dev, &pd->ntmr); - mthca_free(&dev->pd_table.alloc, pd->pd_num); -} - -int mthca_init_pd_table(struct mthca_dev *dev) -{ - return mthca_alloc_init(&dev->pd_table.alloc, - dev->limits.num_pds, - (1 << 24) - 1, - dev->limits.reserved_pds); -} - -void mthca_cleanup_pd_table(struct mthca_dev *dev) -{ - /* XXX check if any PDs are still allocated? */ - mthca_alloc_cleanup(&dev->pd_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.c deleted file mode 100644 index 8edb28a9a0e7..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.c +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include "mthca_profile.h" - -enum { - MTHCA_RES_QP, - MTHCA_RES_EEC, - MTHCA_RES_SRQ, - MTHCA_RES_CQ, - MTHCA_RES_EQP, - MTHCA_RES_EEEC, - MTHCA_RES_EQ, - MTHCA_RES_RDB, - MTHCA_RES_MCG, - MTHCA_RES_MPT, - MTHCA_RES_MTT, - MTHCA_RES_UAR, - MTHCA_RES_UDAV, - MTHCA_RES_UARC, - MTHCA_RES_NUM -}; - -enum { - MTHCA_NUM_EQS = 32, - MTHCA_NUM_PDS = 1 << 15 -}; - -s64 mthca_make_profile(struct mthca_dev *dev, - struct mthca_profile *request, - struct mthca_dev_lim *dev_lim, - struct mthca_init_hca_param *init_hca) -{ - struct mthca_resource { - u64 size; - u64 start; - int type; - int num; - int log_num; - }; - - u64 mem_base, mem_avail; - s64 total_size = 0; - struct mthca_resource *profile; - struct mthca_resource tmp; - int i, j; - - profile = kzalloc(MTHCA_RES_NUM * sizeof *profile, GFP_KERNEL); - if (!profile) - return -ENOMEM; - - profile[MTHCA_RES_QP].size = dev_lim->qpc_entry_sz; - profile[MTHCA_RES_EEC].size = dev_lim->eec_entry_sz; - profile[MTHCA_RES_SRQ].size = dev_lim->srq_entry_sz; - profile[MTHCA_RES_CQ].size = dev_lim->cqc_entry_sz; - profile[MTHCA_RES_EQP].size = dev_lim->eqpc_entry_sz; - profile[MTHCA_RES_EEEC].size = dev_lim->eeec_entry_sz; - profile[MTHCA_RES_EQ].size = dev_lim->eqc_entry_sz; - profile[MTHCA_RES_RDB].size = MTHCA_RDB_ENTRY_SIZE; - profile[MTHCA_RES_MCG].size = MTHCA_MGM_ENTRY_SIZE; - profile[MTHCA_RES_MPT].size = dev_lim->mpt_entry_sz; - profile[MTHCA_RES_MTT].size = dev->limits.mtt_seg_size; - profile[MTHCA_RES_UAR].size = dev_lim->uar_scratch_entry_sz; - profile[MTHCA_RES_UDAV].size = MTHCA_AV_SIZE; - profile[MTHCA_RES_UARC].size = request->uarc_size; - - profile[MTHCA_RES_QP].num = request->num_qp; - profile[MTHCA_RES_SRQ].num = request->num_srq; - profile[MTHCA_RES_EQP].num = request->num_qp; - profile[MTHCA_RES_RDB].num = request->num_qp * request->rdb_per_qp; - profile[MTHCA_RES_CQ].num = request->num_cq; - profile[MTHCA_RES_EQ].num = MTHCA_NUM_EQS; - profile[MTHCA_RES_MCG].num = request->num_mcg; - profile[MTHCA_RES_MPT].num = request->num_mpt; - profile[MTHCA_RES_MTT].num = request->num_mtt; - profile[MTHCA_RES_UAR].num = request->num_uar; - profile[MTHCA_RES_UARC].num = request->num_uar; - profile[MTHCA_RES_UDAV].num = request->num_udav; - - for (i = 0; i < MTHCA_RES_NUM; ++i) { - profile[i].type = i; - profile[i].log_num = max(ffs(profile[i].num) - 1, 0); - profile[i].size *= profile[i].num; - if (mthca_is_memfree(dev)) - profile[i].size = max(profile[i].size, (u64) PAGE_SIZE); - } - - if (mthca_is_memfree(dev)) { - mem_base = 0; - mem_avail = dev_lim->hca.arbel.max_icm_sz; - } else { - mem_base = dev->ddr_start; - mem_avail = dev->fw.tavor.fw_start - dev->ddr_start; - } - - /* - * Sort the resources in decreasing order of size. Since they - * all have sizes that are powers of 2, we'll be able to keep - * resources aligned to their size and pack them without gaps - * using the sorted order. - */ - for (i = MTHCA_RES_NUM; i > 0; --i) - for (j = 1; j < i; ++j) { - if (profile[j].size > profile[j - 1].size) { - tmp = profile[j]; - profile[j] = profile[j - 1]; - profile[j - 1] = tmp; - } - } - - for (i = 0; i < MTHCA_RES_NUM; ++i) { - if (profile[i].size) { - profile[i].start = mem_base + total_size; - total_size += profile[i].size; - } - if (total_size > mem_avail) { - mthca_err(dev, "Profile requires 0x%llx bytes; " - "won't fit in 0x%llx bytes of context memory.\n", - (unsigned long long) total_size, - (unsigned long long) mem_avail); - kfree(profile); - return -ENOMEM; - } - - if (profile[i].size) - mthca_dbg(dev, "profile[%2d]--%2d/%2d @ 0x%16llx " - "(size 0x%8llx)\n", - i, profile[i].type, profile[i].log_num, - (unsigned long long) profile[i].start, - (unsigned long long) profile[i].size); - } - - if (mthca_is_memfree(dev)) - mthca_dbg(dev, "HCA context memory: reserving %d KB\n", - (int) (total_size >> 10)); - else - mthca_dbg(dev, "HCA memory: allocated %d KB/%d KB (%d KB free)\n", - (int) (total_size >> 10), (int) (mem_avail >> 10), - (int) ((mem_avail - total_size) >> 10)); - - for (i = 0; i < MTHCA_RES_NUM; ++i) { - switch (profile[i].type) { - case MTHCA_RES_QP: - dev->limits.num_qps = profile[i].num; - init_hca->qpc_base = profile[i].start; - init_hca->log_num_qps = profile[i].log_num; - break; - case MTHCA_RES_EEC: - dev->limits.num_eecs = profile[i].num; - init_hca->eec_base = profile[i].start; - init_hca->log_num_eecs = profile[i].log_num; - break; - case MTHCA_RES_SRQ: - dev->limits.num_srqs = profile[i].num; - init_hca->srqc_base = profile[i].start; - init_hca->log_num_srqs = profile[i].log_num; - break; - case MTHCA_RES_CQ: - dev->limits.num_cqs = profile[i].num; - init_hca->cqc_base = profile[i].start; - init_hca->log_num_cqs = profile[i].log_num; - break; - case MTHCA_RES_EQP: - init_hca->eqpc_base = profile[i].start; - break; - case MTHCA_RES_EEEC: - init_hca->eeec_base = profile[i].start; - break; - case MTHCA_RES_EQ: - dev->limits.num_eqs = profile[i].num; - init_hca->eqc_base = profile[i].start; - init_hca->log_num_eqs = profile[i].log_num; - break; - case MTHCA_RES_RDB: - for (dev->qp_table.rdb_shift = 0; - request->num_qp << dev->qp_table.rdb_shift < profile[i].num; - ++dev->qp_table.rdb_shift) - ; /* nothing */ - dev->qp_table.rdb_base = (u32) profile[i].start; - init_hca->rdb_base = profile[i].start; - break; - case MTHCA_RES_MCG: - dev->limits.num_mgms = profile[i].num >> 1; - dev->limits.num_amgms = profile[i].num >> 1; - init_hca->mc_base = profile[i].start; - init_hca->log_mc_entry_sz = ffs(MTHCA_MGM_ENTRY_SIZE) - 1; - init_hca->log_mc_table_sz = profile[i].log_num; - init_hca->mc_hash_sz = 1 << (profile[i].log_num - 1); - break; - case MTHCA_RES_MPT: - dev->limits.num_mpts = profile[i].num; - dev->mr_table.mpt_base = profile[i].start; - init_hca->mpt_base = profile[i].start; - init_hca->log_mpt_sz = profile[i].log_num; - break; - case MTHCA_RES_MTT: - dev->limits.num_mtt_segs = profile[i].num; - dev->mr_table.mtt_base = profile[i].start; - init_hca->mtt_base = profile[i].start; - init_hca->mtt_seg_sz = ffs(dev->limits.mtt_seg_size) - 7; - break; - case MTHCA_RES_UAR: - dev->limits.num_uars = profile[i].num; - init_hca->uar_scratch_base = profile[i].start; - break; - case MTHCA_RES_UDAV: - dev->av_table.ddr_av_base = profile[i].start; - dev->av_table.num_ddr_avs = profile[i].num; - break; - case MTHCA_RES_UARC: - dev->uar_table.uarc_size = request->uarc_size; - dev->uar_table.uarc_base = profile[i].start; - init_hca->uarc_base = profile[i].start; - init_hca->log_uarc_sz = ffs(request->uarc_size) - 13; - init_hca->log_uar_sz = ffs(request->num_uar) - 1; - break; - default: - break; - } - } - - /* - * PDs don't take any HCA memory, but we assign them as part - * of the HCA profile anyway. - */ - dev->limits.num_pds = MTHCA_NUM_PDS; - - if (dev->mthca_flags & MTHCA_FLAG_SINAI_OPT && - init_hca->log_mpt_sz > 23) { - mthca_warn(dev, "MPT table too large (requested size 2^%d >= 2^24)\n", - init_hca->log_mpt_sz); - mthca_warn(dev, "Disabling memory key throughput optimization.\n"); - dev->mthca_flags &= ~MTHCA_FLAG_SINAI_OPT; - } - - /* - * For Tavor, FMRs use ioremapped PCI memory. For 32 bit - * systems it may use too much vmalloc space to map all MTT - * memory, so we reserve some MTTs for FMR access, taking them - * out of the MR pool. They don't use additional memory, but - * we assign them as part of the HCA profile anyway. - */ - if (mthca_is_memfree(dev) || BITS_PER_LONG == 64) - dev->limits.fmr_reserved_mtts = 0; - else - dev->limits.fmr_reserved_mtts = request->fmr_reserved_mtts; - - kfree(profile); - return total_size; -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.h deleted file mode 100644 index 62b009cc8730..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_profile.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_PROFILE_H -#define MTHCA_PROFILE_H - -#include "mthca_dev.h" -#include "mthca_cmd.h" - -struct mthca_profile { - int num_qp; - int rdb_per_qp; - int num_srq; - int num_cq; - int num_mcg; - int num_mpt; - int num_mtt; - int num_udav; - int num_uar; - int uarc_size; - int fmr_reserved_mtts; -}; - -s64 mthca_make_profile(struct mthca_dev *mdev, - struct mthca_profile *request, - struct mthca_dev_lim *dev_lim, - struct mthca_init_hca_param *init_hca); - -#endif /* MTHCA_PROFILE_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c deleted file mode 100644 index 264db51e7758..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.c +++ /dev/null @@ -1,1448 +0,0 @@ -/* - * Copyright (c) 2004, 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_user.h" -#include "mthca_memfree.h" - -static void init_query_mad(struct ib_smp *mad) -{ - mad->base_version = 1; - mad->mgmt_class = IB_MGMT_CLASS_SUBN_LID_ROUTED; - mad->class_version = 1; - mad->method = IB_MGMT_METHOD_GET; -} - -static int mthca_query_device(struct ib_device *ibdev, - struct ib_device_attr *props) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - struct mthca_dev *mdev = to_mdev(ibdev); - - u8 status; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - memset(props, 0, sizeof *props); - - props->fw_ver = mdev->fw_ver; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - - err = mthca_MAD_IFC(mdev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - props->device_cap_flags = mdev->device_cap_flags; - props->vendor_id = be32_to_cpup((__be32 *) (out_mad->data + 36)) & - 0xffffff; - props->vendor_part_id = be16_to_cpup((__be16 *) (out_mad->data + 30)); - props->hw_ver = be32_to_cpup((__be32 *) (out_mad->data + 32)); - memcpy(&props->sys_image_guid, out_mad->data + 4, 8); - - props->max_mr_size = ~0ull; - props->page_size_cap = mdev->limits.page_size_cap; - props->max_qp = mdev->limits.num_qps - mdev->limits.reserved_qps; - props->max_qp_wr = mdev->limits.max_wqes; - props->max_sge = mdev->limits.max_sg; - props->max_cq = mdev->limits.num_cqs - mdev->limits.reserved_cqs; - props->max_cqe = mdev->limits.max_cqes; - props->max_mr = mdev->limits.num_mpts - mdev->limits.reserved_mrws; - props->max_pd = mdev->limits.num_pds - mdev->limits.reserved_pds; - props->max_qp_rd_atom = 1 << mdev->qp_table.rdb_shift; - props->max_qp_init_rd_atom = mdev->limits.max_qp_init_rdma; - props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp; - props->max_srq = mdev->limits.num_srqs - mdev->limits.reserved_srqs; - props->max_srq_wr = mdev->limits.max_srq_wqes; - props->max_srq_sge = mdev->limits.max_srq_sge; - props->local_ca_ack_delay = mdev->limits.local_ca_ack_delay; - props->atomic_cap = mdev->limits.flags & DEV_LIM_FLAG_ATOMIC ? - IB_ATOMIC_HCA : IB_ATOMIC_NONE; - props->max_pkeys = mdev->limits.pkey_table_len; - props->max_mcast_grp = mdev->limits.num_mgms + mdev->limits.num_amgms; - props->max_mcast_qp_attach = MTHCA_QP_PER_MGM; - props->max_total_mcast_qp_attach = props->max_mcast_qp_attach * - props->max_mcast_grp; - /* - * If Sinai memory key optimization is being used, then only - * the 8-bit key portion will change. For other HCAs, the - * unused index bits will also be used for FMR remapping. - */ - if (mdev->mthca_flags & MTHCA_FLAG_SINAI_OPT) - props->max_map_per_fmr = 255; - else - props->max_map_per_fmr = - (1 << (32 - ilog2(mdev->limits.num_mpts))) - 1; - - err = 0; - out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int mthca_query_port(struct ib_device *ibdev, - u8 port, struct ib_port_attr *props) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - u8 status; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - memset(props, 0, sizeof *props); - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); - - err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - props->lid = be16_to_cpup((__be16 *) (out_mad->data + 16)); - props->lmc = out_mad->data[34] & 0x7; - props->sm_lid = be16_to_cpup((__be16 *) (out_mad->data + 18)); - props->sm_sl = out_mad->data[36] & 0xf; - props->state = out_mad->data[32] & 0xf; - props->phys_state = out_mad->data[33] >> 4; - props->port_cap_flags = be32_to_cpup((__be32 *) (out_mad->data + 20)); - props->gid_tbl_len = to_mdev(ibdev)->limits.gid_table_len; - props->max_msg_sz = 0x80000000; - props->pkey_tbl_len = to_mdev(ibdev)->limits.pkey_table_len; - props->bad_pkey_cntr = be16_to_cpup((__be16 *) (out_mad->data + 46)); - props->qkey_viol_cntr = be16_to_cpup((__be16 *) (out_mad->data + 48)); - props->active_width = out_mad->data[31] & 0xf; - props->active_speed = out_mad->data[35] >> 4; - props->max_mtu = out_mad->data[41] & 0xf; - props->active_mtu = out_mad->data[36] >> 4; - props->subnet_timeout = out_mad->data[51] & 0x1f; - props->max_vl_num = out_mad->data[37] >> 4; - props->init_type_reply = out_mad->data[41] >> 4; - - out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int mthca_modify_device(struct ib_device *ibdev, - int mask, - struct ib_device_modify *props) -{ - if (mask & ~IB_DEVICE_MODIFY_NODE_DESC) - return -EOPNOTSUPP; - - if (mask & IB_DEVICE_MODIFY_NODE_DESC) { - if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) - return -ERESTARTSYS; - memcpy(ibdev->node_desc, props->node_desc, 64); - mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); - } - - return 0; -} - -static int mthca_modify_port(struct ib_device *ibdev, - u8 port, int port_modify_mask, - struct ib_port_modify *props) -{ - struct mthca_set_ib_param set_ib; - struct ib_port_attr attr; - int err; - u8 status; - - if (mutex_lock_interruptible(&to_mdev(ibdev)->cap_mask_mutex)) - return -ERESTARTSYS; - - err = mthca_query_port(ibdev, port, &attr); - if (err) - goto out; - - set_ib.set_si_guid = 0; - set_ib.reset_qkey_viol = !!(port_modify_mask & IB_PORT_RESET_QKEY_CNTR); - - set_ib.cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) & - ~props->clr_port_cap_mask; - - err = mthca_SET_IB(to_mdev(ibdev), &set_ib, port, &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - -out: - mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex); - return err; -} - -static int mthca_query_pkey(struct ib_device *ibdev, - u8 port, u16 index, u16 *pkey) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - u8 status; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE; - in_mad->attr_mod = cpu_to_be32(index / 32); - - err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]); - - out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int mthca_query_gid(struct ib_device *ibdev, u8 port, - int index, union ib_gid *gid) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - u8 status; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_PORT_INFO; - in_mad->attr_mod = cpu_to_be32(port); - - err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - memcpy(gid->raw, out_mad->data + 8, 8); - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_GUID_INFO; - in_mad->attr_mod = cpu_to_be32(index / 8); - - err = mthca_MAD_IFC(to_mdev(ibdev), 1, 1, - port, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8); - - out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static struct ib_ucontext *mthca_alloc_ucontext(struct ib_device *ibdev, - struct ib_udata *udata) -{ - struct mthca_alloc_ucontext_resp uresp; - struct mthca_ucontext *context; - int err; - - if (!(to_mdev(ibdev)->active)) - return ERR_PTR(-EAGAIN); - - memset(&uresp, 0, sizeof uresp); - - uresp.qp_tab_size = to_mdev(ibdev)->limits.num_qps; - if (mthca_is_memfree(to_mdev(ibdev))) - uresp.uarc_size = to_mdev(ibdev)->uar_table.uarc_size; - else - uresp.uarc_size = 0; - - context = kmalloc(sizeof *context, GFP_KERNEL); - if (!context) - return ERR_PTR(-ENOMEM); - - err = mthca_uar_alloc(to_mdev(ibdev), &context->uar); - if (err) { - kfree(context); - return ERR_PTR(err); - } - - context->db_tab = mthca_init_user_db_tab(to_mdev(ibdev)); - if (IS_ERR(context->db_tab)) { - err = PTR_ERR(context->db_tab); - mthca_uar_free(to_mdev(ibdev), &context->uar); - kfree(context); - return ERR_PTR(err); - } - - if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { - mthca_cleanup_user_db_tab(to_mdev(ibdev), &context->uar, context->db_tab); - mthca_uar_free(to_mdev(ibdev), &context->uar); - kfree(context); - return ERR_PTR(-EFAULT); - } - - context->reg_mr_warned = 0; - - return &context->ibucontext; -} - -static int mthca_dealloc_ucontext(struct ib_ucontext *context) -{ - mthca_cleanup_user_db_tab(to_mdev(context->device), &to_mucontext(context)->uar, - to_mucontext(context)->db_tab); - mthca_uar_free(to_mdev(context->device), &to_mucontext(context)->uar); - kfree(to_mucontext(context)); - - return 0; -} - -static int mthca_mmap_uar(struct ib_ucontext *context, - struct vm_area_struct *vma) -{ - if (vma->vm_end - vma->vm_start != PAGE_SIZE) - return -EINVAL; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (io_remap_pfn_range(vma, vma->vm_start, - to_mucontext(context)->uar.pfn, - PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - - return 0; -} - -static struct ib_pd *mthca_alloc_pd(struct ib_device *ibdev, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mthca_pd *pd; - int err; - - pd = kmalloc(sizeof *pd, GFP_KERNEL); - if (!pd) - return ERR_PTR(-ENOMEM); - - err = mthca_pd_alloc(to_mdev(ibdev), !context, pd); - if (err) { - kfree(pd); - return ERR_PTR(err); - } - - if (context) { - if (ib_copy_to_udata(udata, &pd->pd_num, sizeof (__u32))) { - mthca_pd_free(to_mdev(ibdev), pd); - kfree(pd); - return ERR_PTR(-EFAULT); - } - } - - return &pd->ibpd; -} - -static int mthca_dealloc_pd(struct ib_pd *pd) -{ - mthca_pd_free(to_mdev(pd->device), to_mpd(pd)); - kfree(pd); - - return 0; -} - -static struct ib_ah *mthca_ah_create(struct ib_pd *pd, - struct ib_ah_attr *ah_attr) -{ - int err; - struct mthca_ah *ah; - - ah = kmalloc(sizeof *ah, GFP_ATOMIC); - if (!ah) - return ERR_PTR(-ENOMEM); - - err = mthca_create_ah(to_mdev(pd->device), to_mpd(pd), ah_attr, ah); - if (err) { - kfree(ah); - return ERR_PTR(err); - } - - return &ah->ibah; -} - -static int mthca_ah_destroy(struct ib_ah *ah) -{ - mthca_destroy_ah(to_mdev(ah->device), to_mah(ah)); - kfree(ah); - - return 0; -} - -static struct ib_srq *mthca_create_srq(struct ib_pd *pd, - struct ib_srq_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mthca_create_srq ucmd; - struct mthca_ucontext *context = NULL; - struct mthca_srq *srq; - int err; - - srq = kmalloc(sizeof *srq, GFP_KERNEL); - if (!srq) - return ERR_PTR(-ENOMEM); - - if (pd->uobject) { - context = to_mucontext(pd->uobject->context); - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - err = -EFAULT; - goto err_free; - } - - err = mthca_map_user_db(to_mdev(pd->device), &context->uar, - context->db_tab, ucmd.db_index, - ucmd.db_page); - - if (err) - goto err_free; - - srq->mr.ibmr.lkey = ucmd.lkey; - srq->db_index = ucmd.db_index; - } - - err = mthca_alloc_srq(to_mdev(pd->device), to_mpd(pd), - &init_attr->attr, srq); - - if (err && pd->uobject) - mthca_unmap_user_db(to_mdev(pd->device), &context->uar, - context->db_tab, ucmd.db_index); - - if (err) - goto err_free; - - if (context && ib_copy_to_udata(udata, &srq->srqn, sizeof (__u32))) { - mthca_free_srq(to_mdev(pd->device), srq); - err = -EFAULT; - goto err_free; - } - - return &srq->ibsrq; - -err_free: - kfree(srq); - - return ERR_PTR(err); -} - -static int mthca_destroy_srq(struct ib_srq *srq) -{ - struct mthca_ucontext *context; - - if (srq->uobject) { - context = to_mucontext(srq->uobject->context); - - mthca_unmap_user_db(to_mdev(srq->device), &context->uar, - context->db_tab, to_msrq(srq)->db_index); - } - - mthca_free_srq(to_mdev(srq->device), to_msrq(srq)); - kfree(srq); - - return 0; -} - -static struct ib_qp *mthca_create_qp(struct ib_pd *pd, - struct ib_qp_init_attr *init_attr, - struct ib_udata *udata) -{ - struct mthca_create_qp ucmd; - struct mthca_qp *qp; - int err; - - if (init_attr->create_flags) - return ERR_PTR(-EINVAL); - - switch (init_attr->qp_type) { - case IB_QPT_RC: - case IB_QPT_UC: - case IB_QPT_UD: - { - struct mthca_ucontext *context; - - qp = kmalloc(sizeof *qp, GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - - if (pd->uobject) { - context = to_mucontext(pd->uobject->context); - - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - kfree(qp); - return ERR_PTR(-EFAULT); - } - - err = mthca_map_user_db(to_mdev(pd->device), &context->uar, - context->db_tab, - ucmd.sq_db_index, ucmd.sq_db_page); - if (err) { - kfree(qp); - return ERR_PTR(err); - } - - err = mthca_map_user_db(to_mdev(pd->device), &context->uar, - context->db_tab, - ucmd.rq_db_index, ucmd.rq_db_page); - if (err) { - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, - context->db_tab, - ucmd.sq_db_index); - kfree(qp); - return ERR_PTR(err); - } - - qp->mr.ibmr.lkey = ucmd.lkey; - qp->sq.db_index = ucmd.sq_db_index; - qp->rq.db_index = ucmd.rq_db_index; - } - - err = mthca_alloc_qp(to_mdev(pd->device), to_mpd(pd), - to_mcq(init_attr->send_cq), - to_mcq(init_attr->recv_cq), - init_attr->qp_type, init_attr->sq_sig_type, - &init_attr->cap, qp); - - if (err && pd->uobject) { - context = to_mucontext(pd->uobject->context); - - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, - context->db_tab, - ucmd.sq_db_index); - mthca_unmap_user_db(to_mdev(pd->device), - &context->uar, - context->db_tab, - ucmd.rq_db_index); - } - - qp->ibqp.qp_num = qp->qpn; - break; - } - case IB_QPT_SMI: - case IB_QPT_GSI: - { - /* Don't allow userspace to create special QPs */ - if (pd->uobject) - return ERR_PTR(-EINVAL); - - qp = kmalloc(sizeof (struct mthca_sqp), GFP_KERNEL); - if (!qp) - return ERR_PTR(-ENOMEM); - - qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1; - - err = mthca_alloc_sqp(to_mdev(pd->device), to_mpd(pd), - to_mcq(init_attr->send_cq), - to_mcq(init_attr->recv_cq), - init_attr->sq_sig_type, &init_attr->cap, - qp->ibqp.qp_num, init_attr->port_num, - to_msqp(qp)); - break; - } - default: - /* Don't support raw QPs */ - return ERR_PTR(-ENOSYS); - } - - if (err) { - kfree(qp); - return ERR_PTR(err); - } - - init_attr->cap.max_send_wr = qp->sq.max; - init_attr->cap.max_recv_wr = qp->rq.max; - init_attr->cap.max_send_sge = qp->sq.max_gs; - init_attr->cap.max_recv_sge = qp->rq.max_gs; - init_attr->cap.max_inline_data = qp->max_inline_data; - - return &qp->ibqp; -} - -static int mthca_destroy_qp(struct ib_qp *qp) -{ - if (qp->uobject) { - mthca_unmap_user_db(to_mdev(qp->device), - &to_mucontext(qp->uobject->context)->uar, - to_mucontext(qp->uobject->context)->db_tab, - to_mqp(qp)->sq.db_index); - mthca_unmap_user_db(to_mdev(qp->device), - &to_mucontext(qp->uobject->context)->uar, - to_mucontext(qp->uobject->context)->db_tab, - to_mqp(qp)->rq.db_index); - } - mthca_free_qp(to_mdev(qp->device), to_mqp(qp)); - kfree(qp); - return 0; -} - -static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, - struct ib_cq_init_attr *attr, - struct ib_ucontext *context, - struct ib_udata *udata) -{ - struct mthca_create_cq ucmd; - struct mthca_cq *cq; - int nent; - int err; - int entries = attr->cqe; - - if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) - return ERR_PTR(-EINVAL); - - if (context) { - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) - return ERR_PTR(-EFAULT); - - err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, - to_mucontext(context)->db_tab, - ucmd.set_db_index, ucmd.set_db_page); - if (err) - return ERR_PTR(err); - - err = mthca_map_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, - to_mucontext(context)->db_tab, - ucmd.arm_db_index, ucmd.arm_db_page); - if (err) - goto err_unmap_set; - } - - cq = kmalloc(sizeof *cq, GFP_KERNEL); - if (!cq) { - err = -ENOMEM; - goto err_unmap_arm; - } - - if (context) { - cq->buf.mr.ibmr.lkey = ucmd.lkey; - cq->set_ci_db_index = ucmd.set_db_index; - cq->arm_db_index = ucmd.arm_db_index; - } - - for (nent = 1; nent <= entries; nent <<= 1) - ; /* nothing */ - - err = mthca_init_cq(to_mdev(ibdev), nent, - context ? to_mucontext(context) : NULL, - context ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, - cq); - if (err) - goto err_free; - - if (context && ib_copy_to_udata(udata, &cq->cqn, sizeof (__u32))) { - mthca_free_cq(to_mdev(ibdev), cq); - goto err_free; - } - - cq->resize_buf = NULL; - - return &cq->ibcq; - -err_free: - kfree(cq); - -err_unmap_arm: - if (context) - mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, - to_mucontext(context)->db_tab, ucmd.arm_db_index); - -err_unmap_set: - if (context) - mthca_unmap_user_db(to_mdev(ibdev), &to_mucontext(context)->uar, - to_mucontext(context)->db_tab, ucmd.set_db_index); - - return ERR_PTR(err); -} - -static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, - int entries) -{ - int ret; - - spin_lock_irq(&cq->lock); - if (cq->resize_buf) { - ret = -EBUSY; - goto unlock; - } - - cq->resize_buf = kmalloc(sizeof *cq->resize_buf, GFP_ATOMIC); - if (!cq->resize_buf) { - ret = -ENOMEM; - goto unlock; - } - - cq->resize_buf->state = CQ_RESIZE_ALLOC; - - ret = 0; - -unlock: - spin_unlock_irq(&cq->lock); - - if (ret) - return ret; - - ret = mthca_alloc_cq_buf(dev, &cq->resize_buf->buf, entries); - if (ret) { - spin_lock_irq(&cq->lock); - kfree(cq->resize_buf); - cq->resize_buf = NULL; - spin_unlock_irq(&cq->lock); - return ret; - } - - cq->resize_buf->cqe = entries - 1; - - spin_lock_irq(&cq->lock); - cq->resize_buf->state = CQ_RESIZE_READY; - spin_unlock_irq(&cq->lock); - - return 0; -} - -static int mthca_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata) -{ - struct mthca_dev *dev = to_mdev(ibcq->device); - struct mthca_cq *cq = to_mcq(ibcq); - struct mthca_resize_cq ucmd; - u32 lkey; - u8 status; - int ret; - - if (entries < 1 || entries > dev->limits.max_cqes) - return -EINVAL; - - mutex_lock(&cq->mutex); - - entries = roundup_pow_of_two(entries + 1); - if (entries == ibcq->cqe + 1) { - ret = 0; - goto out; - } - - if (cq->is_kernel) { - ret = mthca_alloc_resize_buf(dev, cq, entries); - if (ret) - goto out; - lkey = cq->resize_buf->buf.mr.ibmr.lkey; - } else { - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) { - ret = -EFAULT; - goto out; - } - lkey = ucmd.lkey; - } - - ret = mthca_RESIZE_CQ(dev, cq->cqn, lkey, ilog2(entries), &status); - if (status) - ret = -EINVAL; - - if (ret) { - if (cq->resize_buf) { - mthca_free_cq_buf(dev, &cq->resize_buf->buf, - cq->resize_buf->cqe); - kfree(cq->resize_buf); - spin_lock_irq(&cq->lock); - cq->resize_buf = NULL; - spin_unlock_irq(&cq->lock); - } - goto out; - } - - if (cq->is_kernel) { - struct mthca_cq_buf tbuf; - int tcqe; - - spin_lock_irq(&cq->lock); - if (cq->resize_buf->state == CQ_RESIZE_READY) { - mthca_cq_resize_copy_cqes(cq); - tbuf = cq->buf; - tcqe = cq->ibcq.cqe; - cq->buf = cq->resize_buf->buf; - cq->ibcq.cqe = cq->resize_buf->cqe; - } else { - tbuf = cq->resize_buf->buf; - tcqe = cq->resize_buf->cqe; - } - - kfree(cq->resize_buf); - cq->resize_buf = NULL; - spin_unlock_irq(&cq->lock); - - mthca_free_cq_buf(dev, &tbuf, tcqe); - } else - ibcq->cqe = entries - 1; - -out: - mutex_unlock(&cq->mutex); - - return ret; -} - -static int mthca_destroy_cq(struct ib_cq *cq) -{ - if (cq->uobject) { - mthca_unmap_user_db(to_mdev(cq->device), - &to_mucontext(cq->uobject->context)->uar, - to_mucontext(cq->uobject->context)->db_tab, - to_mcq(cq)->arm_db_index); - mthca_unmap_user_db(to_mdev(cq->device), - &to_mucontext(cq->uobject->context)->uar, - to_mucontext(cq->uobject->context)->db_tab, - to_mcq(cq)->set_ci_db_index); - } - mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); - kfree(cq); - - return 0; -} - -static inline u32 convert_access(int acc) -{ - return (acc & IB_ACCESS_REMOTE_ATOMIC ? MTHCA_MPT_FLAG_ATOMIC : 0) | - (acc & IB_ACCESS_REMOTE_WRITE ? MTHCA_MPT_FLAG_REMOTE_WRITE : 0) | - (acc & IB_ACCESS_REMOTE_READ ? MTHCA_MPT_FLAG_REMOTE_READ : 0) | - (acc & IB_ACCESS_LOCAL_WRITE ? MTHCA_MPT_FLAG_LOCAL_WRITE : 0) | - MTHCA_MPT_FLAG_LOCAL_READ; -} - -static struct ib_mr *mthca_get_dma_mr(struct ib_pd *pd, int acc) -{ - struct mthca_mr *mr; - int err; - - mr = kmalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - err = mthca_mr_alloc_notrans(to_mdev(pd->device), - to_mpd(pd)->pd_num, - convert_access(acc), mr); - - if (err) { - kfree(mr); - return ERR_PTR(err); - } - - mr->umem = NULL; - - return &mr->ibmr; -} - -static struct ib_mr *mthca_reg_phys_mr(struct ib_pd *pd, - struct ib_phys_buf *buffer_list, - int num_phys_buf, - int acc, - u64 *iova_start) -{ - struct mthca_mr *mr; - u64 *page_list; - u64 total_size; - unsigned long mask; - int shift; - int npages; - int err; - int i, j, n; - - mask = buffer_list[0].addr ^ *iova_start; - total_size = 0; - for (i = 0; i < num_phys_buf; ++i) { - if (i != 0) - mask |= buffer_list[i].addr; - if (i != num_phys_buf - 1) - mask |= buffer_list[i].addr + buffer_list[i].size; - - total_size += buffer_list[i].size; - } - - if (mask & ~PAGE_MASK) - return ERR_PTR(-EINVAL); - - shift = __ffs(mask | 1 << 31); - - buffer_list[0].size += buffer_list[0].addr & ((1ULL << shift) - 1); - buffer_list[0].addr &= ~0ull << shift; - - mr = kmalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - npages = 0; - for (i = 0; i < num_phys_buf; ++i) - npages += (buffer_list[i].size + (1ULL << shift) - 1) >> shift; - - if (!npages) - return &mr->ibmr; - - page_list = kmalloc(npages * sizeof *page_list, GFP_KERNEL); - if (!page_list) { - kfree(mr); - return ERR_PTR(-ENOMEM); - } - - n = 0; - for (i = 0; i < num_phys_buf; ++i) - for (j = 0; - j < (buffer_list[i].size + (1ULL << shift) - 1) >> shift; - ++j) - page_list[n++] = buffer_list[i].addr + ((u64) j << shift); - - mthca_dbg(to_mdev(pd->device), "Registering memory at %llx (iova %llx) " - "in PD %x; shift %d, npages %d.\n", - (unsigned long long) buffer_list[0].addr, - (unsigned long long) *iova_start, - to_mpd(pd)->pd_num, - shift, npages); - - err = mthca_mr_alloc_phys(to_mdev(pd->device), - to_mpd(pd)->pd_num, - page_list, shift, npages, - *iova_start, total_size, - convert_access(acc), mr); - - if (err) { - kfree(page_list); - kfree(mr); - return ERR_PTR(err); - } - - kfree(page_list); - mr->umem = NULL; - - return &mr->ibmr; -} - -static struct ib_mr *mthca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata, int mr_id) -{ - struct mthca_dev *dev = to_mdev(pd->device); - struct scatterlist *sg; - struct mthca_mr *mr; - struct mthca_reg_mr ucmd; - u64 *pages; - int shift, n, len; - int i, k, entry; - int err = 0; - int write_mtt_size; - - if (udata->inlen - sizeof (struct ib_uverbs_cmd_hdr) < sizeof ucmd) { - if (!to_mucontext(pd->uobject->context)->reg_mr_warned) { - mthca_warn(dev, "Process '%s' did not pass in MR attrs.\n", - curproc->p_comm); - mthca_warn(dev, " Update libmthca to fix this.\n"); - } - ++to_mucontext(pd->uobject->context)->reg_mr_warned; - ucmd.mr_attrs = 0; - } else if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) - return ERR_PTR(-EFAULT); - - mr = kmalloc(sizeof *mr, GFP_KERNEL); - if (!mr) - return ERR_PTR(-ENOMEM); - - mr->umem = ib_umem_get(pd->uobject->context, start, length, acc, - ucmd.mr_attrs & MTHCA_MR_DMASYNC); - - if (IS_ERR(mr->umem)) { - err = PTR_ERR(mr->umem); - goto err; - } - - shift = ffs(mr->umem->page_size) - 1; - - n = mr->umem->nmap; - mr->mtt = mthca_alloc_mtt(dev, n); - if (IS_ERR(mr->mtt)) { - err = PTR_ERR(mr->mtt); - goto err_umem; - } - - pages = (u64 *) __get_free_page(GFP_KERNEL); - if (!pages) { - err = -ENOMEM; - goto err_mtt; - } - - i = n = 0; - - write_mtt_size = min(mthca_write_mtt_size(dev), (int) (PAGE_SIZE / sizeof *pages)); - - for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) { - len = sg_dma_len(sg) >> shift; - for (k = 0; k < len; ++k) { - pages[i++] = sg_dma_address(sg) + - mr->umem->page_size * k; - /* - * Be friendly to write_mtt and pass it chunks - * of appropriate size. - */ - if (i == write_mtt_size) { - err = mthca_write_mtt(dev, mr->mtt, n, pages, i); - if (err) - goto mtt_done; - n += i; - i = 0; - } - } - } - - - - - if (i) - err = mthca_write_mtt(dev, mr->mtt, n, pages, i); -mtt_done: - free_page((unsigned long) pages); - if (err) - goto err_mtt; - - err = mthca_mr_alloc(dev, to_mpd(pd)->pd_num, shift, virt, length, - convert_access(acc), mr); - - if (err) - goto err_mtt; - - return &mr->ibmr; - -err_mtt: - mthca_free_mtt(dev, mr->mtt); - -err_umem: - ib_umem_release(mr->umem); - -err: - kfree(mr); - return ERR_PTR(err); -} - -static int mthca_dereg_mr(struct ib_mr *mr) -{ - struct mthca_mr *mmr = to_mmr(mr); - - mthca_free_mr(to_mdev(mr->device), mmr); - if (mmr->umem) - ib_umem_release(mmr->umem); - kfree(mmr); - - return 0; -} - -static struct ib_fmr *mthca_alloc_fmr(struct ib_pd *pd, int mr_access_flags, - struct ib_fmr_attr *fmr_attr) -{ - struct mthca_fmr *fmr; - int err; - - fmr = kmalloc(sizeof *fmr, GFP_KERNEL); - if (!fmr) - return ERR_PTR(-ENOMEM); - - memcpy(&fmr->attr, fmr_attr, sizeof *fmr_attr); - err = mthca_fmr_alloc(to_mdev(pd->device), to_mpd(pd)->pd_num, - convert_access(mr_access_flags), fmr); - - if (err) { - kfree(fmr); - return ERR_PTR(err); - } - - return &fmr->ibmr; -} - -static int mthca_dealloc_fmr(struct ib_fmr *fmr) -{ - struct mthca_fmr *mfmr = to_mfmr(fmr); - int err; - - err = mthca_free_fmr(to_mdev(fmr->device), mfmr); - if (err) - return err; - - kfree(mfmr); - return 0; -} - -static int mthca_unmap_fmr(struct list_head *fmr_list) -{ - struct ib_fmr *fmr; - int err; - u8 status; - struct mthca_dev *mdev = NULL; - - list_for_each_entry(fmr, fmr_list, list) { - if (mdev && to_mdev(fmr->device) != mdev) - return -EINVAL; - mdev = to_mdev(fmr->device); - } - - if (!mdev) - return 0; - - if (mthca_is_memfree(mdev)) { - list_for_each_entry(fmr, fmr_list, list) - mthca_arbel_fmr_unmap(mdev, to_mfmr(fmr)); - - wmb(); - } else - list_for_each_entry(fmr, fmr_list, list) - mthca_tavor_fmr_unmap(mdev, to_mfmr(fmr)); - - err = mthca_SYNC_TPT(mdev, &status); - if (err) - return err; - if (status) - return -EINVAL; - return 0; -} - -static ssize_t show_rev(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mthca_dev *dev = - container_of(device, struct mthca_dev, ib_dev.dev); - return sprintf(buf, "%x\n", dev->rev_id); -} - -static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mthca_dev *dev = - container_of(device, struct mthca_dev, ib_dev.dev); - return sprintf(buf, "%d.%d.%d\n", (int) (dev->fw_ver >> 32), - (int) (dev->fw_ver >> 16) & 0xffff, - (int) dev->fw_ver & 0xffff); -} - -static ssize_t show_hca(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mthca_dev *dev = - container_of(device, struct mthca_dev, ib_dev.dev); - switch (dev->pdev->device) { - case PCI_DEVICE_ID_MELLANOX_TAVOR: - return sprintf(buf, "MT23108\n"); - case PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT: - return sprintf(buf, "MT25208 (MT23108 compat mode)\n"); - case PCI_DEVICE_ID_MELLANOX_ARBEL: - return sprintf(buf, "MT25208\n"); - case PCI_DEVICE_ID_MELLANOX_SINAI: - case PCI_DEVICE_ID_MELLANOX_SINAI_OLD: - return sprintf(buf, "MT25204\n"); - default: - return sprintf(buf, "unknown\n"); - } -} - -static ssize_t show_board(struct device *device, struct device_attribute *attr, - char *buf) -{ - struct mthca_dev *dev = - container_of(device, struct mthca_dev, ib_dev.dev); - return sprintf(buf, "%.*s\n", MTHCA_BOARD_ID_LEN, dev->board_id); -} - -static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL); -static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL); -static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL); -static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL); - -static struct device_attribute *mthca_dev_attributes[] = { - &dev_attr_hw_rev, - &dev_attr_fw_ver, - &dev_attr_hca_type, - &dev_attr_board_id -}; - -static int mthca_init_node_data(struct mthca_dev *dev) -{ - struct ib_smp *in_mad = NULL; - struct ib_smp *out_mad = NULL; - int err = -ENOMEM; - u8 status; - - in_mad = kzalloc(sizeof *in_mad, GFP_KERNEL); - out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL); - if (!in_mad || !out_mad) - goto out; - - init_query_mad(in_mad); - in_mad->attr_id = IB_SMP_ATTR_NODE_DESC; - - err = mthca_MAD_IFC(dev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - memcpy(dev->ib_dev.node_desc, out_mad->data, 64); - - in_mad->attr_id = IB_SMP_ATTR_NODE_INFO; - - err = mthca_MAD_IFC(dev, 1, 1, - 1, NULL, NULL, in_mad, out_mad, - &status); - if (err) - goto out; - if (status) { - err = -EINVAL; - goto out; - } - - if (mthca_is_memfree(dev)) - dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32)); - memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8); - -out: - kfree(in_mad); - kfree(out_mad); - return err; -} - -static int mthca_port_immutable(struct ib_device *ibdev, u8 port_num, - struct ib_port_immutable *immutable) -{ - struct ib_port_attr attr; - int err; - - immutable->core_cap_flags = RDMA_CORE_PORT_IBA_IB; - - err = ib_query_port(ibdev, port_num, &attr); - if (err) - return err; - - immutable->pkey_tbl_len = attr.pkey_tbl_len; - immutable->gid_tbl_len = attr.gid_tbl_len; - immutable->max_mad_size = IB_MGMT_MAD_SIZE; - - return 0; -} - -int mthca_register_device(struct mthca_dev *dev) -{ - int ret; - int i; - - ret = mthca_init_node_data(dev); - if (ret) - return ret; - - strlcpy(dev->ib_dev.name, "mthca%d", IB_DEVICE_NAME_MAX); - dev->ib_dev.owner = THIS_MODULE; - - dev->ib_dev.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION; - dev->ib_dev.uverbs_cmd_mask = - (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | - (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | - (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | - (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_REG_MR) | - (1ull << IB_USER_VERBS_CMD_DEREG_MR) | - (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | - (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | - (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_QP) | - (1ull << IB_USER_VERBS_CMD_QUERY_QP) | - (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | - (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | - (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) | - (1ull << IB_USER_VERBS_CMD_DETACH_MCAST); - dev->ib_dev.node_type = RDMA_NODE_IB_CA; - dev->ib_dev.phys_port_cnt = dev->limits.num_ports; - dev->ib_dev.num_comp_vectors = 1; - dev->ib_dev.dma_device = &dev->pdev->dev; - dev->ib_dev.query_device = mthca_query_device; - dev->ib_dev.query_port = mthca_query_port; - dev->ib_dev.modify_device = mthca_modify_device; - dev->ib_dev.modify_port = mthca_modify_port; - dev->ib_dev.query_pkey = mthca_query_pkey; - dev->ib_dev.query_gid = mthca_query_gid; - dev->ib_dev.alloc_ucontext = mthca_alloc_ucontext; - dev->ib_dev.dealloc_ucontext = mthca_dealloc_ucontext; - dev->ib_dev.mmap = mthca_mmap_uar; - dev->ib_dev.alloc_pd = mthca_alloc_pd; - dev->ib_dev.dealloc_pd = mthca_dealloc_pd; - dev->ib_dev.create_ah = mthca_ah_create; - dev->ib_dev.query_ah = mthca_ah_query; - dev->ib_dev.destroy_ah = mthca_ah_destroy; - - if (dev->mthca_flags & MTHCA_FLAG_SRQ) { - dev->ib_dev.create_srq = mthca_create_srq; - dev->ib_dev.modify_srq = mthca_modify_srq; - dev->ib_dev.query_srq = mthca_query_srq; - dev->ib_dev.destroy_srq = mthca_destroy_srq; - dev->ib_dev.uverbs_cmd_mask |= - (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) | - (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) | - (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ); - - if (mthca_is_memfree(dev)) - dev->ib_dev.post_srq_recv = mthca_arbel_post_srq_recv; - else - dev->ib_dev.post_srq_recv = mthca_tavor_post_srq_recv; - } - - dev->ib_dev.create_qp = mthca_create_qp; - dev->ib_dev.modify_qp = mthca_modify_qp; - dev->ib_dev.query_qp = mthca_query_qp; - dev->ib_dev.destroy_qp = mthca_destroy_qp; - dev->ib_dev.create_cq = mthca_create_cq; - dev->ib_dev.resize_cq = mthca_resize_cq; - dev->ib_dev.destroy_cq = mthca_destroy_cq; - dev->ib_dev.poll_cq = mthca_poll_cq; - dev->ib_dev.get_dma_mr = mthca_get_dma_mr; - dev->ib_dev.reg_phys_mr = mthca_reg_phys_mr; - dev->ib_dev.reg_user_mr = mthca_reg_user_mr; - dev->ib_dev.dereg_mr = mthca_dereg_mr; - dev->ib_dev.get_port_immutable = mthca_port_immutable; - - if (dev->mthca_flags & MTHCA_FLAG_FMR) { - dev->ib_dev.alloc_fmr = mthca_alloc_fmr; - dev->ib_dev.unmap_fmr = mthca_unmap_fmr; - dev->ib_dev.dealloc_fmr = mthca_dealloc_fmr; - if (mthca_is_memfree(dev)) - dev->ib_dev.map_phys_fmr = mthca_arbel_map_phys_fmr; - else - dev->ib_dev.map_phys_fmr = mthca_tavor_map_phys_fmr; - } - - dev->ib_dev.attach_mcast = mthca_multicast_attach; - dev->ib_dev.detach_mcast = mthca_multicast_detach; - dev->ib_dev.process_mad = mthca_process_mad; - - if (mthca_is_memfree(dev)) { - dev->ib_dev.req_notify_cq = mthca_arbel_arm_cq; - dev->ib_dev.post_send = mthca_arbel_post_send; - dev->ib_dev.post_recv = mthca_arbel_post_receive; - } else { - dev->ib_dev.req_notify_cq = mthca_tavor_arm_cq; - dev->ib_dev.post_send = mthca_tavor_post_send; - dev->ib_dev.post_recv = mthca_tavor_post_receive; - } - - mutex_init(&dev->cap_mask_mutex); - - ret = ib_register_device(&dev->ib_dev, NULL); - if (ret) - return ret; - - for (i = 0; i < ARRAY_SIZE(mthca_dev_attributes); ++i) { - ret = device_create_file(&dev->ib_dev.dev, - mthca_dev_attributes[i]); - if (ret) { - ib_unregister_device(&dev->ib_dev); - return ret; - } - } - - mthca_start_catas_poll(dev); - - return 0; -} - -void mthca_unregister_device(struct mthca_dev *dev) -{ - mthca_stop_catas_poll(dev); - ib_unregister_device(&dev->ib_dev); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.h deleted file mode 100644 index c621f8794b88..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_provider.h +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_PROVIDER_H -#define MTHCA_PROVIDER_H - -#include -#include - -#define MTHCA_MPT_FLAG_ATOMIC (1 << 14) -#define MTHCA_MPT_FLAG_REMOTE_WRITE (1 << 13) -#define MTHCA_MPT_FLAG_REMOTE_READ (1 << 12) -#define MTHCA_MPT_FLAG_LOCAL_WRITE (1 << 11) -#define MTHCA_MPT_FLAG_LOCAL_READ (1 << 10) - -struct mthca_buf_list { - void *buf; - DECLARE_PCI_UNMAP_ADDR(mapping) -}; - -union mthca_buf { - struct mthca_buf_list direct; - struct mthca_buf_list *page_list; -}; - -struct mthca_uar { - unsigned long pfn; - int index; -}; - -struct mthca_user_db_table; - -struct mthca_ucontext { - struct ib_ucontext ibucontext; - struct mthca_uar uar; - struct mthca_user_db_table *db_tab; - int reg_mr_warned; -}; - -struct mthca_mtt; - -struct mthca_mr { - struct ib_mr ibmr; - struct ib_umem *umem; - struct mthca_mtt *mtt; -}; - -struct mthca_fmr { - struct ib_fmr ibmr; - struct ib_fmr_attr attr; - struct mthca_mtt *mtt; - int maps; - union { - struct { - struct mthca_mpt_entry __iomem *mpt; - u64 __iomem *mtts; - } tavor; - struct { - struct mthca_mpt_entry *mpt; - __be64 *mtts; - dma_addr_t dma_handle; - } arbel; - } mem; -}; - -struct mthca_pd { - struct ib_pd ibpd; - u32 pd_num; - atomic_t sqp_count; - struct mthca_mr ntmr; - int privileged; -}; - -struct mthca_eq { - struct mthca_dev *dev; - int eqn; - u32 eqn_mask; - u32 cons_index; - u16 msi_x_vector; - u16 msi_x_entry; - int have_irq; - int nent; - struct mthca_buf_list *page_list; - struct mthca_mr mr; -}; - -struct mthca_av; - -enum mthca_ah_type { - MTHCA_AH_ON_HCA, - MTHCA_AH_PCI_POOL, - MTHCA_AH_KMALLOC -}; - -struct mthca_ah { - struct ib_ah ibah; - enum mthca_ah_type type; - u32 key; - struct mthca_av *av; - dma_addr_t avdma; -}; - -/* - * Quick description of our CQ/QP locking scheme: - * - * We have one global lock that protects dev->cq/qp_table. Each - * struct mthca_cq/qp also has its own lock. An individual qp lock - * may be taken inside of an individual cq lock. Both cqs attached to - * a qp may be locked, with the cq with the lower cqn locked first. - * No other nesting should be done. - * - * Each struct mthca_cq/qp also has an ref count, protected by the - * corresponding table lock. The pointer from the cq/qp_table to the - * struct counts as one reference. This reference also is good for - * access through the consumer API, so modifying the CQ/QP etc doesn't - * need to take another reference. Access to a QP because of a - * completion being polled does not need a reference either. - * - * Finally, each struct mthca_cq/qp has a wait_queue_head_t for the - * destroy function to sleep on. - * - * This means that access from the consumer API requires nothing but - * taking the struct's lock. - * - * Access because of a completion event should go as follows: - * - lock cq/qp_table and look up struct - * - increment ref count in struct - * - drop cq/qp_table lock - * - lock struct, do your thing, and unlock struct - * - decrement ref count; if zero, wake up waiters - * - * To destroy a CQ/QP, we can do the following: - * - lock cq/qp_table - * - remove pointer and decrement ref count - * - unlock cq/qp_table lock - * - wait_event until ref count is zero - * - * It is the consumer's responsibilty to make sure that no QP - * operations (WQE posting or state modification) are pending when a - * QP is destroyed. Also, the consumer must make sure that calls to - * qp_modify are serialized. Similarly, the consumer is responsible - * for ensuring that no CQ resize operations are pending when a CQ - * is destroyed. - * - * Possible optimizations (wait for profile data to see if/where we - * have locks bouncing between CPUs): - * - split cq/qp table lock into n separate (cache-aligned) locks, - * indexed (say) by the page in the table - * - split QP struct lock into three (one for common info, one for the - * send queue and one for the receive queue) - */ - -struct mthca_cq_buf { - union mthca_buf queue; - struct mthca_mr mr; - int is_direct; -}; - -struct mthca_cq_resize { - struct mthca_cq_buf buf; - int cqe; - enum { - CQ_RESIZE_ALLOC, - CQ_RESIZE_READY, - CQ_RESIZE_SWAPPED - } state; -}; - -struct mthca_cq { - struct ib_cq ibcq; - spinlock_t lock; - int refcount; - int cqn; - u32 cons_index; - struct mthca_cq_buf buf; - struct mthca_cq_resize *resize_buf; - int is_kernel; - - /* Next fields are Arbel only */ - int set_ci_db_index; - __be32 *set_ci_db; - int arm_db_index; - __be32 *arm_db; - int arm_sn; - - wait_queue_head_t wait; - struct mutex mutex; -}; - -struct mthca_srq { - struct ib_srq ibsrq; - spinlock_t lock; - int refcount; - int srqn; - int max; - int max_gs; - int wqe_shift; - int first_free; - int last_free; - u16 counter; /* Arbel only */ - int db_index; /* Arbel only */ - __be32 *db; /* Arbel only */ - void *last; - - int is_direct; - u64 *wrid; - union mthca_buf queue; - struct mthca_mr mr; - - wait_queue_head_t wait; - struct mutex mutex; -}; - -struct mthca_wq { - spinlock_t lock; - int max; - unsigned next_ind; - unsigned last_comp; - unsigned head; - unsigned tail; - void *last; - int max_gs; - int wqe_shift; - - int db_index; /* Arbel only */ - __be32 *db; -}; - -struct mthca_qp { - struct ib_qp ibqp; - int refcount; - u32 qpn; - int is_direct; - u8 port; /* for SQP and memfree use only */ - u8 alt_port; /* for memfree use only */ - u8 transport; - u8 state; - u8 atomic_rd_en; - u8 resp_depth; - - struct mthca_mr mr; - - struct mthca_wq rq; - struct mthca_wq sq; - enum ib_sig_type sq_policy; - int send_wqe_offset; - int max_inline_data; - - u64 *wrid; - union mthca_buf queue; - - wait_queue_head_t wait; - struct mutex mutex; -}; - -struct mthca_sqp { - struct mthca_qp qp; - int pkey_index; - u32 qkey; - u32 send_psn; - struct ib_ud_header ud_header; - int header_buf_size; - void *header_buf; - dma_addr_t header_dma; -}; - -static inline struct mthca_ucontext *to_mucontext(struct ib_ucontext *ibucontext) -{ - return container_of(ibucontext, struct mthca_ucontext, ibucontext); -} - -static inline struct mthca_fmr *to_mfmr(struct ib_fmr *ibmr) -{ - return container_of(ibmr, struct mthca_fmr, ibmr); -} - -static inline struct mthca_mr *to_mmr(struct ib_mr *ibmr) -{ - return container_of(ibmr, struct mthca_mr, ibmr); -} - -static inline struct mthca_pd *to_mpd(struct ib_pd *ibpd) -{ - return container_of(ibpd, struct mthca_pd, ibpd); -} - -static inline struct mthca_ah *to_mah(struct ib_ah *ibah) -{ - return container_of(ibah, struct mthca_ah, ibah); -} - -static inline struct mthca_cq *to_mcq(struct ib_cq *ibcq) -{ - return container_of(ibcq, struct mthca_cq, ibcq); -} - -static inline struct mthca_srq *to_msrq(struct ib_srq *ibsrq) -{ - return container_of(ibsrq, struct mthca_srq, ibsrq); -} - -static inline struct mthca_qp *to_mqp(struct ib_qp *ibqp) -{ - return container_of(ibqp, struct mthca_qp, ibqp); -} - -static inline struct mthca_sqp *to_msqp(struct mthca_qp *qp) -{ - return container_of(qp, struct mthca_sqp, qp); -} - -#endif /* MTHCA_PROVIDER_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_qp.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_qp.c deleted file mode 100644 index b4c70b415317..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_qp.c +++ /dev/null @@ -1,2333 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * Copyright (c) 2005 Mellanox Technologies. All rights reserved. - * Copyright (c) 2004 Voltaire, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include - -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_memfree.h" -#include "mthca_wqe.h" - -enum { - MTHCA_MAX_DIRECT_QP_SIZE = 4 * PAGE_SIZE, - MTHCA_ACK_REQ_FREQ = 10, - MTHCA_FLIGHT_LIMIT = 9, - MTHCA_UD_HEADER_SIZE = 72, /* largest UD header possible */ - MTHCA_INLINE_HEADER_SIZE = 4, /* data segment overhead for inline */ - MTHCA_INLINE_CHUNK_SIZE = 16 /* inline data segment chunk */ -}; - -enum { - MTHCA_QP_STATE_RST = 0, - MTHCA_QP_STATE_INIT = 1, - MTHCA_QP_STATE_RTR = 2, - MTHCA_QP_STATE_RTS = 3, - MTHCA_QP_STATE_SQE = 4, - MTHCA_QP_STATE_SQD = 5, - MTHCA_QP_STATE_ERR = 6, - MTHCA_QP_STATE_DRAINING = 7 -}; - -enum { - MTHCA_QP_ST_RC = 0x0, - MTHCA_QP_ST_UC = 0x1, - MTHCA_QP_ST_RD = 0x2, - MTHCA_QP_ST_UD = 0x3, - MTHCA_QP_ST_MLX = 0x7 -}; - -enum { - MTHCA_QP_PM_MIGRATED = 0x3, - MTHCA_QP_PM_ARMED = 0x0, - MTHCA_QP_PM_REARM = 0x1 -}; - -enum { - /* qp_context flags */ - MTHCA_QP_BIT_DE = 1 << 8, - /* params1 */ - MTHCA_QP_BIT_SRE = 1 << 15, - MTHCA_QP_BIT_SWE = 1 << 14, - MTHCA_QP_BIT_SAE = 1 << 13, - MTHCA_QP_BIT_SIC = 1 << 4, - MTHCA_QP_BIT_SSC = 1 << 3, - /* params2 */ - MTHCA_QP_BIT_RRE = 1 << 15, - MTHCA_QP_BIT_RWE = 1 << 14, - MTHCA_QP_BIT_RAE = 1 << 13, - MTHCA_QP_BIT_RIC = 1 << 4, - MTHCA_QP_BIT_RSC = 1 << 3 -}; - -enum { - MTHCA_SEND_DOORBELL_FENCE = 1 << 5 -}; - -struct mthca_qp_path { - __be32 port_pkey; - u8 rnr_retry; - u8 g_mylmc; - __be16 rlid; - u8 ackto; - u8 mgid_index; - u8 static_rate; - u8 hop_limit; - __be32 sl_tclass_flowlabel; - u8 rgid[16]; -} __attribute__((packed)); - -struct mthca_qp_context { - __be32 flags; - __be32 tavor_sched_queue; /* Reserved on Arbel */ - u8 mtu_msgmax; - u8 rq_size_stride; /* Reserved on Tavor */ - u8 sq_size_stride; /* Reserved on Tavor */ - u8 rlkey_arbel_sched_queue; /* Reserved on Tavor */ - __be32 usr_page; - __be32 local_qpn; - __be32 remote_qpn; - u32 reserved1[2]; - struct mthca_qp_path pri_path; - struct mthca_qp_path alt_path; - __be32 rdd; - __be32 pd; - __be32 wqe_base; - __be32 wqe_lkey; - __be32 params1; - __be32 reserved2; - __be32 next_send_psn; - __be32 cqn_snd; - __be32 snd_wqe_base_l; /* Next send WQE on Tavor */ - __be32 snd_db_index; /* (debugging only entries) */ - __be32 last_acked_psn; - __be32 ssn; - __be32 params2; - __be32 rnr_nextrecvpsn; - __be32 ra_buff_indx; - __be32 cqn_rcv; - __be32 rcv_wqe_base_l; /* Next recv WQE on Tavor */ - __be32 rcv_db_index; /* (debugging only entries) */ - __be32 qkey; - __be32 srqn; - __be32 rmsn; - __be16 rq_wqe_counter; /* reserved on Tavor */ - __be16 sq_wqe_counter; /* reserved on Tavor */ - u32 reserved3[18]; -} __attribute__((packed)); - -struct mthca_qp_param { - __be32 opt_param_mask; - u32 reserved1; - struct mthca_qp_context context; - u32 reserved2[62]; -} __attribute__((packed)); - -enum { - MTHCA_QP_OPTPAR_ALT_ADDR_PATH = 1 << 0, - MTHCA_QP_OPTPAR_RRE = 1 << 1, - MTHCA_QP_OPTPAR_RAE = 1 << 2, - MTHCA_QP_OPTPAR_RWE = 1 << 3, - MTHCA_QP_OPTPAR_PKEY_INDEX = 1 << 4, - MTHCA_QP_OPTPAR_Q_KEY = 1 << 5, - MTHCA_QP_OPTPAR_RNR_TIMEOUT = 1 << 6, - MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH = 1 << 7, - MTHCA_QP_OPTPAR_SRA_MAX = 1 << 8, - MTHCA_QP_OPTPAR_RRA_MAX = 1 << 9, - MTHCA_QP_OPTPAR_PM_STATE = 1 << 10, - MTHCA_QP_OPTPAR_PORT_NUM = 1 << 11, - MTHCA_QP_OPTPAR_RETRY_COUNT = 1 << 12, - MTHCA_QP_OPTPAR_ALT_RNR_RETRY = 1 << 13, - MTHCA_QP_OPTPAR_ACK_TIMEOUT = 1 << 14, - MTHCA_QP_OPTPAR_RNR_RETRY = 1 << 15, - MTHCA_QP_OPTPAR_SCHED_QUEUE = 1 << 16 -}; - -static const u8 mthca_opcode[] = { - [IB_WR_SEND] = MTHCA_OPCODE_SEND, - [IB_WR_SEND_WITH_IMM] = MTHCA_OPCODE_SEND_IMM, - [IB_WR_RDMA_WRITE] = MTHCA_OPCODE_RDMA_WRITE, - [IB_WR_RDMA_WRITE_WITH_IMM] = MTHCA_OPCODE_RDMA_WRITE_IMM, - [IB_WR_RDMA_READ] = MTHCA_OPCODE_RDMA_READ, - [IB_WR_ATOMIC_CMP_AND_SWP] = MTHCA_OPCODE_ATOMIC_CS, - [IB_WR_ATOMIC_FETCH_AND_ADD] = MTHCA_OPCODE_ATOMIC_FA, -}; - -static int is_sqp(struct mthca_dev *dev, struct mthca_qp *qp) -{ - return qp->qpn >= dev->qp_table.sqp_start && - qp->qpn <= dev->qp_table.sqp_start + 3; -} - -static int is_qp0(struct mthca_dev *dev, struct mthca_qp *qp) -{ - return qp->qpn >= dev->qp_table.sqp_start && - qp->qpn <= dev->qp_table.sqp_start + 1; -} - -static void *get_recv_wqe(struct mthca_qp *qp, int n) -{ - if (qp->is_direct) - return qp->queue.direct.buf + (n << qp->rq.wqe_shift); - else - return qp->queue.page_list[(n << qp->rq.wqe_shift) >> PAGE_SHIFT].buf + - ((n << qp->rq.wqe_shift) & (PAGE_SIZE - 1)); -} - -static void *get_send_wqe(struct mthca_qp *qp, int n) -{ - if (qp->is_direct) - return qp->queue.direct.buf + qp->send_wqe_offset + - (n << qp->sq.wqe_shift); - else - return qp->queue.page_list[(qp->send_wqe_offset + - (n << qp->sq.wqe_shift)) >> - PAGE_SHIFT].buf + - ((qp->send_wqe_offset + (n << qp->sq.wqe_shift)) & - (PAGE_SIZE - 1)); -} - -static void mthca_wq_reset(struct mthca_wq *wq) -{ - wq->next_ind = 0; - wq->last_comp = wq->max - 1; - wq->head = 0; - wq->tail = 0; -} - -void mthca_qp_event(struct mthca_dev *dev, u32 qpn, - enum ib_event_type event_type) -{ - struct mthca_qp *qp; - struct ib_event event; - - spin_lock(&dev->qp_table.lock); - qp = mthca_array_get(&dev->qp_table.qp, qpn & (dev->limits.num_qps - 1)); - if (qp) - ++qp->refcount; - spin_unlock(&dev->qp_table.lock); - - if (!qp) { - mthca_warn(dev, "Async event %d for bogus QP %08x\n", - (int) event_type, qpn); - return; - } - - if (event_type == IB_EVENT_PATH_MIG) - qp->port = qp->alt_port; - - event.device = &dev->ib_dev; - event.event = event_type; - event.element.qp = &qp->ibqp; - if (qp->ibqp.event_handler) - qp->ibqp.event_handler(&event, qp->ibqp.qp_context); - - spin_lock(&dev->qp_table.lock); - if (!--qp->refcount) - wake_up(&qp->wait); - spin_unlock(&dev->qp_table.lock); -} - -static int to_mthca_state(enum ib_qp_state ib_state) -{ - switch (ib_state) { - case IB_QPS_RESET: return MTHCA_QP_STATE_RST; - case IB_QPS_INIT: return MTHCA_QP_STATE_INIT; - case IB_QPS_RTR: return MTHCA_QP_STATE_RTR; - case IB_QPS_RTS: return MTHCA_QP_STATE_RTS; - case IB_QPS_SQD: return MTHCA_QP_STATE_SQD; - case IB_QPS_SQE: return MTHCA_QP_STATE_SQE; - case IB_QPS_ERR: return MTHCA_QP_STATE_ERR; - default: return -1; - } -} - -enum { RC, UC, UD, RD, RDEE, MLX, NUM_TRANS }; - -static int to_mthca_st(int transport) -{ - switch (transport) { - case RC: return MTHCA_QP_ST_RC; - case UC: return MTHCA_QP_ST_UC; - case UD: return MTHCA_QP_ST_UD; - case RD: return MTHCA_QP_ST_RD; - case MLX: return MTHCA_QP_ST_MLX; - default: return -1; - } -} - -static void store_attrs(struct mthca_sqp *sqp, const struct ib_qp_attr *attr, - int attr_mask) -{ - if (attr_mask & IB_QP_PKEY_INDEX) - sqp->pkey_index = attr->pkey_index; - if (attr_mask & IB_QP_QKEY) - sqp->qkey = attr->qkey; - if (attr_mask & IB_QP_SQ_PSN) - sqp->send_psn = attr->sq_psn; -} - -static void init_port(struct mthca_dev *dev, int port) -{ - int err; - u8 status; - struct mthca_init_ib_param param; - - memset(¶m, 0, sizeof param); - - param.port_width = dev->limits.port_width_cap; - param.vl_cap = dev->limits.vl_cap; - param.mtu_cap = dev->limits.mtu_cap; - param.gid_cap = dev->limits.gid_table_len; - param.pkey_cap = dev->limits.pkey_table_len; - - err = mthca_INIT_IB(dev, ¶m, port, &status); - if (err) - mthca_warn(dev, "INIT_IB failed, return code %d.\n", err); - if (status) - mthca_warn(dev, "INIT_IB returned status %02x.\n", status); -} - -static __be32 get_hw_access_flags(struct mthca_qp *qp, const struct ib_qp_attr *attr, - int attr_mask) -{ - u8 dest_rd_atomic; - u32 access_flags; - u32 hw_access_flags = 0; - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - dest_rd_atomic = attr->max_dest_rd_atomic; - else - dest_rd_atomic = qp->resp_depth; - - if (attr_mask & IB_QP_ACCESS_FLAGS) - access_flags = attr->qp_access_flags; - else - access_flags = qp->atomic_rd_en; - - if (!dest_rd_atomic) - access_flags &= IB_ACCESS_REMOTE_WRITE; - - if (access_flags & IB_ACCESS_REMOTE_READ) - hw_access_flags |= MTHCA_QP_BIT_RRE; - if (access_flags & IB_ACCESS_REMOTE_ATOMIC) - hw_access_flags |= MTHCA_QP_BIT_RAE; - if (access_flags & IB_ACCESS_REMOTE_WRITE) - hw_access_flags |= MTHCA_QP_BIT_RWE; - - return cpu_to_be32(hw_access_flags); -} - -static inline enum ib_qp_state to_ib_qp_state(int mthca_state) -{ - switch (mthca_state) { - case MTHCA_QP_STATE_RST: return IB_QPS_RESET; - case MTHCA_QP_STATE_INIT: return IB_QPS_INIT; - case MTHCA_QP_STATE_RTR: return IB_QPS_RTR; - case MTHCA_QP_STATE_RTS: return IB_QPS_RTS; - case MTHCA_QP_STATE_DRAINING: - case MTHCA_QP_STATE_SQD: return IB_QPS_SQD; - case MTHCA_QP_STATE_SQE: return IB_QPS_SQE; - case MTHCA_QP_STATE_ERR: return IB_QPS_ERR; - default: return -1; - } -} - -static inline enum ib_mig_state to_ib_mig_state(int mthca_mig_state) -{ - switch (mthca_mig_state) { - case 0: return IB_MIG_ARMED; - case 1: return IB_MIG_REARM; - case 3: return IB_MIG_MIGRATED; - default: return -1; - } -} - -static int to_ib_qp_access_flags(int mthca_flags) -{ - int ib_flags = 0; - - if (mthca_flags & MTHCA_QP_BIT_RRE) - ib_flags |= IB_ACCESS_REMOTE_READ; - if (mthca_flags & MTHCA_QP_BIT_RWE) - ib_flags |= IB_ACCESS_REMOTE_WRITE; - if (mthca_flags & MTHCA_QP_BIT_RAE) - ib_flags |= IB_ACCESS_REMOTE_ATOMIC; - - return ib_flags; -} - -static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr, - struct mthca_qp_path *path) -{ - memset(ib_ah_attr, 0, sizeof *ib_ah_attr); - ib_ah_attr->port_num = (be32_to_cpu(path->port_pkey) >> 24) & 0x3; - - if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->limits.num_ports) - return; - - ib_ah_attr->dlid = be16_to_cpu(path->rlid); - ib_ah_attr->sl = be32_to_cpu(path->sl_tclass_flowlabel) >> 28; - ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f; - ib_ah_attr->static_rate = mthca_rate_to_ib(dev, - path->static_rate & 0xf, - ib_ah_attr->port_num); - ib_ah_attr->ah_flags = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0; - if (ib_ah_attr->ah_flags) { - ib_ah_attr->grh.sgid_index = path->mgid_index & (dev->limits.gid_table_len - 1); - ib_ah_attr->grh.hop_limit = path->hop_limit; - ib_ah_attr->grh.traffic_class = - (be32_to_cpu(path->sl_tclass_flowlabel) >> 20) & 0xff; - ib_ah_attr->grh.flow_label = - be32_to_cpu(path->sl_tclass_flowlabel) & 0xfffff; - memcpy(ib_ah_attr->grh.dgid.raw, - path->rgid, sizeof ib_ah_attr->grh.dgid.raw); - } -} - -int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask, - struct ib_qp_init_attr *qp_init_attr) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - int err = 0; - struct mthca_mailbox *mailbox = NULL; - struct mthca_qp_param *qp_param; - struct mthca_qp_context *context; - int mthca_state; - u8 status; - - mutex_lock(&qp->mutex); - - if (qp->state == IB_QPS_RESET) { - qp_attr->qp_state = IB_QPS_RESET; - goto done; - } - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto out; - } - - err = mthca_QUERY_QP(dev, qp->qpn, 0, mailbox, &status); - if (err) - goto out_mailbox; - if (status) { - mthca_warn(dev, "QUERY_QP returned status %02x\n", status); - err = -EINVAL; - goto out_mailbox; - } - - qp_param = mailbox->buf; - context = &qp_param->context; - mthca_state = be32_to_cpu(context->flags) >> 28; - - qp->state = to_ib_qp_state(mthca_state); - qp_attr->qp_state = qp->state; - qp_attr->path_mtu = context->mtu_msgmax >> 5; - qp_attr->path_mig_state = - to_ib_mig_state((be32_to_cpu(context->flags) >> 11) & 0x3); - qp_attr->qkey = be32_to_cpu(context->qkey); - qp_attr->rq_psn = be32_to_cpu(context->rnr_nextrecvpsn) & 0xffffff; - qp_attr->sq_psn = be32_to_cpu(context->next_send_psn) & 0xffffff; - qp_attr->dest_qp_num = be32_to_cpu(context->remote_qpn) & 0xffffff; - qp_attr->qp_access_flags = - to_ib_qp_access_flags(be32_to_cpu(context->params2)); - - if (qp->transport == RC || qp->transport == UC) { - to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path); - to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path); - qp_attr->alt_pkey_index = - be32_to_cpu(context->alt_path.port_pkey) & 0x7f; - qp_attr->alt_port_num = qp_attr->alt_ah_attr.port_num; - } - - qp_attr->pkey_index = be32_to_cpu(context->pri_path.port_pkey) & 0x7f; - qp_attr->port_num = - (be32_to_cpu(context->pri_path.port_pkey) >> 24) & 0x3; - - /* qp_attr->en_sqd_async_notify is only applicable in modify qp */ - qp_attr->sq_draining = mthca_state == MTHCA_QP_STATE_DRAINING; - - qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context->params1) >> 21) & 0x7); - - qp_attr->max_dest_rd_atomic = - 1 << ((be32_to_cpu(context->params2) >> 21) & 0x7); - qp_attr->min_rnr_timer = - (be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f; - qp_attr->timeout = context->pri_path.ackto >> 3; - qp_attr->retry_cnt = (be32_to_cpu(context->params1) >> 16) & 0x7; - qp_attr->rnr_retry = context->pri_path.rnr_retry >> 5; - qp_attr->alt_timeout = context->alt_path.ackto >> 3; - -done: - qp_attr->cur_qp_state = qp_attr->qp_state; - qp_attr->cap.max_send_wr = qp->sq.max; - qp_attr->cap.max_recv_wr = qp->rq.max; - qp_attr->cap.max_send_sge = qp->sq.max_gs; - qp_attr->cap.max_recv_sge = qp->rq.max_gs; - qp_attr->cap.max_inline_data = qp->max_inline_data; - - qp_init_attr->cap = qp_attr->cap; - -out_mailbox: - mthca_free_mailbox(dev, mailbox); - -out: - mutex_unlock(&qp->mutex); - return err; -} - -static int mthca_path_set(struct mthca_dev *dev, const struct ib_ah_attr *ah, - struct mthca_qp_path *path, u8 port) -{ - path->g_mylmc = ah->src_path_bits & 0x7f; - path->rlid = cpu_to_be16(ah->dlid); - path->static_rate = mthca_get_rate(dev, ah->static_rate, port); - - if (ah->ah_flags & IB_AH_GRH) { - if (ah->grh.sgid_index >= dev->limits.gid_table_len) { - mthca_dbg(dev, "sgid_index (%u) too large. max is %d\n", - ah->grh.sgid_index, dev->limits.gid_table_len-1); - return -1; - } - - path->g_mylmc |= 1 << 7; - path->mgid_index = ah->grh.sgid_index; - path->hop_limit = ah->grh.hop_limit; - path->sl_tclass_flowlabel = - cpu_to_be32((ah->sl << 28) | - (ah->grh.traffic_class << 20) | - (ah->grh.flow_label)); - memcpy(path->rgid, ah->grh.dgid.raw, 16); - } else - path->sl_tclass_flowlabel = cpu_to_be32(ah->sl << 28); - - return 0; -} - -static int __mthca_modify_qp(struct ib_qp *ibqp, - const struct ib_qp_attr *attr, int attr_mask, - enum ib_qp_state cur_state, enum ib_qp_state new_state) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - struct mthca_mailbox *mailbox; - struct mthca_qp_param *qp_param; - struct mthca_qp_context *qp_context; - u32 sqd_event = 0; - u8 status; - int err = -EINVAL; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto out; - } - qp_param = mailbox->buf; - qp_context = &qp_param->context; - memset(qp_param, 0, sizeof *qp_param); - - qp_context->flags = cpu_to_be32((to_mthca_state(new_state) << 28) | - (to_mthca_st(qp->transport) << 16)); - qp_context->flags |= cpu_to_be32(MTHCA_QP_BIT_DE); - if (!(attr_mask & IB_QP_PATH_MIG_STATE)) - qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_MIGRATED << 11); - else { - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PM_STATE); - switch (attr->path_mig_state) { - case IB_MIG_MIGRATED: - qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_MIGRATED << 11); - break; - case IB_MIG_REARM: - qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_REARM << 11); - break; - case IB_MIG_ARMED: - qp_context->flags |= cpu_to_be32(MTHCA_QP_PM_ARMED << 11); - break; - } - } - - /* leave tavor_sched_queue as 0 */ - - if (qp->transport == MLX || qp->transport == UD) - qp_context->mtu_msgmax = (IB_MTU_2048 << 5) | 11; - else if (attr_mask & IB_QP_PATH_MTU) { - if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_2048) { - mthca_dbg(dev, "path MTU (%u) is invalid\n", - attr->path_mtu); - goto out_mailbox; - } - qp_context->mtu_msgmax = (attr->path_mtu << 5) | 31; - } - - if (mthca_is_memfree(dev)) { - if (qp->rq.max) - qp_context->rq_size_stride = ilog2(qp->rq.max) << 3; - qp_context->rq_size_stride |= qp->rq.wqe_shift - 4; - - if (qp->sq.max) - qp_context->sq_size_stride = ilog2(qp->sq.max) << 3; - qp_context->sq_size_stride |= qp->sq.wqe_shift - 4; - } - - /* leave arbel_sched_queue as 0 */ - - if (qp->ibqp.uobject) - qp_context->usr_page = - cpu_to_be32(to_mucontext(qp->ibqp.uobject->context)->uar.index); - else - qp_context->usr_page = cpu_to_be32(dev->driver_uar.index); - qp_context->local_qpn = cpu_to_be32(qp->qpn); - if (attr_mask & IB_QP_DEST_QPN) { - qp_context->remote_qpn = cpu_to_be32(attr->dest_qp_num); - } - - if (qp->transport == MLX) - qp_context->pri_path.port_pkey |= - cpu_to_be32(qp->port << 24); - else { - if (attr_mask & IB_QP_PORT) { - qp_context->pri_path.port_pkey |= - cpu_to_be32(attr->port_num << 24); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PORT_NUM); - } - } - - if (attr_mask & IB_QP_PKEY_INDEX) { - qp_context->pri_path.port_pkey |= - cpu_to_be32(attr->pkey_index); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PKEY_INDEX); - } - - if (attr_mask & IB_QP_RNR_RETRY) { - qp_context->alt_path.rnr_retry = qp_context->pri_path.rnr_retry = - attr->rnr_retry << 5; - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RNR_RETRY | - MTHCA_QP_OPTPAR_ALT_RNR_RETRY); - } - - if (attr_mask & IB_QP_AV) { - if (mthca_path_set(dev, &attr->ah_attr, &qp_context->pri_path, - attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) - goto out_mailbox; - - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_PRIMARY_ADDR_PATH); - } - - if (ibqp->qp_type == IB_QPT_RC && - cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) { - u8 sched_queue = ibqp->uobject ? 0x2 : 0x1; - - if (mthca_is_memfree(dev)) - qp_context->rlkey_arbel_sched_queue |= sched_queue; - else - qp_context->tavor_sched_queue |= cpu_to_be32(sched_queue); - - qp_param->opt_param_mask |= - cpu_to_be32(MTHCA_QP_OPTPAR_SCHED_QUEUE); - } - - if (attr_mask & IB_QP_TIMEOUT) { - qp_context->pri_path.ackto = attr->timeout << 3; - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_ACK_TIMEOUT); - } - - if (attr_mask & IB_QP_ALT_PATH) { - if (attr->alt_pkey_index >= dev->limits.pkey_table_len) { - mthca_dbg(dev, "Alternate P_Key index (%u) too large. max is %d\n", - attr->alt_pkey_index, dev->limits.pkey_table_len-1); - goto out_mailbox; - } - - if (attr->alt_port_num == 0 || attr->alt_port_num > dev->limits.num_ports) { - mthca_dbg(dev, "Alternate port number (%u) is invalid\n", - attr->alt_port_num); - goto out_mailbox; - } - - if (mthca_path_set(dev, &attr->alt_ah_attr, &qp_context->alt_path, - attr->alt_ah_attr.port_num)) - goto out_mailbox; - - qp_context->alt_path.port_pkey |= cpu_to_be32(attr->alt_pkey_index | - attr->alt_port_num << 24); - qp_context->alt_path.ackto = attr->alt_timeout << 3; - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_ALT_ADDR_PATH); - } - - /* leave rdd as 0 */ - qp_context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pd_num); - /* leave wqe_base as 0 (we always create an MR based at 0 for WQs) */ - qp_context->wqe_lkey = cpu_to_be32(qp->mr.ibmr.lkey); - qp_context->params1 = cpu_to_be32((MTHCA_ACK_REQ_FREQ << 28) | - (MTHCA_FLIGHT_LIMIT << 24) | - MTHCA_QP_BIT_SWE); - if (qp->sq_policy == IB_SIGNAL_ALL_WR) - qp_context->params1 |= cpu_to_be32(MTHCA_QP_BIT_SSC); - if (attr_mask & IB_QP_RETRY_CNT) { - qp_context->params1 |= cpu_to_be32(attr->retry_cnt << 16); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RETRY_COUNT); - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) { - if (attr->max_rd_atomic) { - qp_context->params1 |= - cpu_to_be32(MTHCA_QP_BIT_SRE | - MTHCA_QP_BIT_SAE); - qp_context->params1 |= - cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21); - } - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_SRA_MAX); - } - - if (attr_mask & IB_QP_SQ_PSN) - qp_context->next_send_psn = cpu_to_be32(attr->sq_psn); - qp_context->cqn_snd = cpu_to_be32(to_mcq(ibqp->send_cq)->cqn); - - if (mthca_is_memfree(dev)) { - qp_context->snd_wqe_base_l = cpu_to_be32(qp->send_wqe_offset); - qp_context->snd_db_index = cpu_to_be32(qp->sq.db_index); - } - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) { - if (attr->max_dest_rd_atomic) - qp_context->params2 |= - cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21); - - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RRA_MAX); - } - - if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) { - qp_context->params2 |= get_hw_access_flags(qp, attr, attr_mask); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RWE | - MTHCA_QP_OPTPAR_RRE | - MTHCA_QP_OPTPAR_RAE); - } - - qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RSC); - - if (ibqp->srq) - qp_context->params2 |= cpu_to_be32(MTHCA_QP_BIT_RIC); - - if (attr_mask & IB_QP_MIN_RNR_TIMER) { - qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_RNR_TIMEOUT); - } - if (attr_mask & IB_QP_RQ_PSN) - qp_context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn); - - qp_context->ra_buff_indx = - cpu_to_be32(dev->qp_table.rdb_base + - ((qp->qpn & (dev->limits.num_qps - 1)) * MTHCA_RDB_ENTRY_SIZE << - dev->qp_table.rdb_shift)); - - qp_context->cqn_rcv = cpu_to_be32(to_mcq(ibqp->recv_cq)->cqn); - - if (mthca_is_memfree(dev)) - qp_context->rcv_db_index = cpu_to_be32(qp->rq.db_index); - - if (attr_mask & IB_QP_QKEY) { - qp_context->qkey = cpu_to_be32(attr->qkey); - qp_param->opt_param_mask |= cpu_to_be32(MTHCA_QP_OPTPAR_Q_KEY); - } - - if (ibqp->srq) - qp_context->srqn = cpu_to_be32(1 << 24 | - to_msrq(ibqp->srq)->srqn); - - if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD && - attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && - attr->en_sqd_async_notify) - sqd_event = 1 << 31; - - err = mthca_MODIFY_QP(dev, cur_state, new_state, qp->qpn, 0, - mailbox, sqd_event, &status); - if (err) - goto out_mailbox; - if (status) { - mthca_warn(dev, "modify QP %d->%d returned status %02x.\n", - cur_state, new_state, status); - err = -EINVAL; - goto out_mailbox; - } - - qp->state = new_state; - if (attr_mask & IB_QP_ACCESS_FLAGS) - qp->atomic_rd_en = attr->qp_access_flags; - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) - qp->resp_depth = attr->max_dest_rd_atomic; - if (attr_mask & IB_QP_PORT) - qp->port = attr->port_num; - if (attr_mask & IB_QP_ALT_PATH) - qp->alt_port = attr->alt_port_num; - - if (is_sqp(dev, qp)) - store_attrs(to_msqp(qp), attr, attr_mask); - - /* - * If we moved QP0 to RTR, bring the IB link up; if we moved - * QP0 to RESET or ERROR, bring the link back down. - */ - if (is_qp0(dev, qp)) { - if (cur_state != IB_QPS_RTR && - new_state == IB_QPS_RTR) - init_port(dev, qp->port); - - if (cur_state != IB_QPS_RESET && - cur_state != IB_QPS_ERR && - (new_state == IB_QPS_RESET || - new_state == IB_QPS_ERR)) - mthca_CLOSE_IB(dev, qp->port, &status); - } - - /* - * If we moved a kernel QP to RESET, clean up all old CQ - * entries and reinitialize the QP. - */ - if (new_state == IB_QPS_RESET && !qp->ibqp.uobject) { - mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq), qp->qpn, - qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); - if (qp->ibqp.send_cq != qp->ibqp.recv_cq) - mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq), qp->qpn, NULL); - - mthca_wq_reset(&qp->sq); - qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); - - mthca_wq_reset(&qp->rq); - qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); - - if (mthca_is_memfree(dev)) { - *qp->sq.db = 0; - *qp->rq.db = 0; - } - } - -out_mailbox: - mthca_free_mailbox(dev, mailbox); -out: - return err; -} - -int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, - struct ib_udata *udata) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - enum ib_qp_state cur_state, new_state; - int err = -EINVAL; - - mutex_lock(&qp->mutex); - if (attr_mask & IB_QP_CUR_STATE) { - cur_state = attr->cur_qp_state; - } else { - spin_lock_irq(&qp->sq.lock); - spin_lock(&qp->rq.lock); - cur_state = qp->state; - spin_unlock(&qp->rq.lock); - spin_unlock_irq(&qp->sq.lock); - } - - new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state; - - if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, - attr_mask, IB_LINK_LAYER_UNSPECIFIED)) { - mthca_dbg(dev, "Bad QP transition (transport %d) " - "%d->%d with attr 0x%08x\n", - qp->transport, cur_state, new_state, - attr_mask); - goto out; - } - - if ((attr_mask & IB_QP_PKEY_INDEX) && - attr->pkey_index >= dev->limits.pkey_table_len) { - mthca_dbg(dev, "P_Key index (%u) too large. max is %d\n", - attr->pkey_index, dev->limits.pkey_table_len-1); - goto out; - } - - if ((attr_mask & IB_QP_PORT) && - (attr->port_num == 0 || attr->port_num > dev->limits.num_ports)) { - mthca_dbg(dev, "Port number (%u) is invalid\n", attr->port_num); - goto out; - } - - if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && - attr->max_rd_atomic > dev->limits.max_qp_init_rdma) { - mthca_dbg(dev, "Max rdma_atomic as initiator %u too large (max is %d)\n", - attr->max_rd_atomic, dev->limits.max_qp_init_rdma); - goto out; - } - - if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && - attr->max_dest_rd_atomic > 1 << dev->qp_table.rdb_shift) { - mthca_dbg(dev, "Max rdma_atomic as responder %u too large (max %d)\n", - attr->max_dest_rd_atomic, 1 << dev->qp_table.rdb_shift); - goto out; - } - - if (cur_state == new_state && cur_state == IB_QPS_RESET) { - err = 0; - goto out; - } - - err = __mthca_modify_qp(ibqp, attr, attr_mask, cur_state, new_state); - -out: - mutex_unlock(&qp->mutex); - return err; -} - -static int mthca_max_data_size(struct mthca_dev *dev, struct mthca_qp *qp, int desc_sz) -{ - /* - * Calculate the maximum size of WQE s/g segments, excluding - * the next segment and other non-data segments. - */ - int max_data_size = desc_sz - sizeof (struct mthca_next_seg); - - switch (qp->transport) { - case MLX: - max_data_size -= 2 * sizeof (struct mthca_data_seg); - break; - - case UD: - if (mthca_is_memfree(dev)) - max_data_size -= sizeof (struct mthca_arbel_ud_seg); - else - max_data_size -= sizeof (struct mthca_tavor_ud_seg); - break; - - default: - max_data_size -= sizeof (struct mthca_raddr_seg); - break; - } - - return max_data_size; -} - -static inline int mthca_max_inline_data(struct mthca_pd *pd, int max_data_size) -{ - /* We don't support inline data for kernel QPs (yet). */ - return pd->ibpd.uobject ? max_data_size - MTHCA_INLINE_HEADER_SIZE : 0; -} - -static void mthca_adjust_qp_caps(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_qp *qp) -{ - int max_data_size = mthca_max_data_size(dev, qp, - min(dev->limits.max_desc_sz, - 1 << qp->sq.wqe_shift)); - - qp->max_inline_data = mthca_max_inline_data(pd, max_data_size); - - qp->sq.max_gs = min_t(int, dev->limits.max_sg, - max_data_size / sizeof (struct mthca_data_seg)); - qp->rq.max_gs = min_t(int, dev->limits.max_sg, - (min(dev->limits.max_desc_sz, 1 << qp->rq.wqe_shift) - - sizeof (struct mthca_next_seg)) / - sizeof (struct mthca_data_seg)); -} - -/* - * Allocate and register buffer for WQEs. qp->rq.max, sq.max, - * rq.max_gs and sq.max_gs must all be assigned. - * mthca_alloc_wqe_buf will calculate rq.wqe_shift and - * sq.wqe_shift (as well as send_wqe_offset, is_direct, and - * queue) - */ -static int mthca_alloc_wqe_buf(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_qp *qp) -{ - int size; - int err = -ENOMEM; - - size = sizeof (struct mthca_next_seg) + - qp->rq.max_gs * sizeof (struct mthca_data_seg); - - if (size > dev->limits.max_desc_sz) - return -EINVAL; - - for (qp->rq.wqe_shift = 6; 1 << qp->rq.wqe_shift < size; - qp->rq.wqe_shift++) - ; /* nothing */ - - size = qp->sq.max_gs * sizeof (struct mthca_data_seg); - switch (qp->transport) { - case MLX: - size += 2 * sizeof (struct mthca_data_seg); - break; - - case UD: - size += mthca_is_memfree(dev) ? - sizeof (struct mthca_arbel_ud_seg) : - sizeof (struct mthca_tavor_ud_seg); - break; - - case UC: - size += sizeof (struct mthca_raddr_seg); - break; - - case RC: - size += sizeof (struct mthca_raddr_seg); - /* - * An atomic op will require an atomic segment, a - * remote address segment and one scatter entry. - */ - size = max_t(int, size, - sizeof (struct mthca_atomic_seg) + - sizeof (struct mthca_raddr_seg) + - sizeof (struct mthca_data_seg)); - break; - - default: - break; - } - - /* Make sure that we have enough space for a bind request */ - size = max_t(int, size, sizeof (struct mthca_bind_seg)); - - size += sizeof (struct mthca_next_seg); - - if (size > dev->limits.max_desc_sz) - return -EINVAL; - - for (qp->sq.wqe_shift = 6; 1 << qp->sq.wqe_shift < size; - qp->sq.wqe_shift++) - ; /* nothing */ - - qp->send_wqe_offset = ALIGN(qp->rq.max << qp->rq.wqe_shift, - 1 << qp->sq.wqe_shift); - - /* - * If this is a userspace QP, we don't actually have to - * allocate anything. All we need is to calculate the WQE - * sizes and the send_wqe_offset, so we're done now. - */ - if (pd->ibpd.uobject) - return 0; - - size = PAGE_ALIGN(qp->send_wqe_offset + - (qp->sq.max << qp->sq.wqe_shift)); - - qp->wrid = kmalloc((qp->rq.max + qp->sq.max) * sizeof (u64), - GFP_KERNEL); - if (!qp->wrid) - goto err_out; - - err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_QP_SIZE, - &qp->queue, &qp->is_direct, pd, 0, &qp->mr); - if (err) - goto err_out; - - return 0; - -err_out: - kfree(qp->wrid); - return err; -} - -static void mthca_free_wqe_buf(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - mthca_buf_free(dev, PAGE_ALIGN(qp->send_wqe_offset + - (qp->sq.max << qp->sq.wqe_shift)), - &qp->queue, qp->is_direct, &qp->mr); - kfree(qp->wrid); -} - -static int mthca_map_memfree(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - int ret; - - if (mthca_is_memfree(dev)) { - ret = mthca_table_get(dev, dev->qp_table.qp_table, qp->qpn); - if (ret) - return ret; - - ret = mthca_table_get(dev, dev->qp_table.eqp_table, qp->qpn); - if (ret) - goto err_qpc; - - ret = mthca_table_get(dev, dev->qp_table.rdb_table, - qp->qpn << dev->qp_table.rdb_shift); - if (ret) - goto err_eqpc; - - } - - return 0; - -err_eqpc: - mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn); - -err_qpc: - mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn); - - return ret; -} - -static void mthca_unmap_memfree(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - mthca_table_put(dev, dev->qp_table.rdb_table, - qp->qpn << dev->qp_table.rdb_shift); - mthca_table_put(dev, dev->qp_table.eqp_table, qp->qpn); - mthca_table_put(dev, dev->qp_table.qp_table, qp->qpn); -} - -static int mthca_alloc_memfree(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - if (mthca_is_memfree(dev)) { - qp->rq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_RQ, - qp->qpn, &qp->rq.db); - if (qp->rq.db_index < 0) - return -ENOMEM; - - qp->sq.db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SQ, - qp->qpn, &qp->sq.db); - if (qp->sq.db_index < 0) { - mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index); - return -ENOMEM; - } - } - - return 0; -} - -static void mthca_free_memfree(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - if (mthca_is_memfree(dev)) { - mthca_free_db(dev, MTHCA_DB_TYPE_SQ, qp->sq.db_index); - mthca_free_db(dev, MTHCA_DB_TYPE_RQ, qp->rq.db_index); - } -} - -static int mthca_alloc_qp_common(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_cq *send_cq, - struct mthca_cq *recv_cq, - enum ib_sig_type send_policy, - struct mthca_qp *qp) -{ - int ret; - int i; - struct mthca_next_seg *next; - - qp->refcount = 1; - init_waitqueue_head(&qp->wait); - mutex_init(&qp->mutex); - qp->state = IB_QPS_RESET; - qp->atomic_rd_en = 0; - qp->resp_depth = 0; - qp->sq_policy = send_policy; - mthca_wq_reset(&qp->sq); - mthca_wq_reset(&qp->rq); - - spin_lock_init(&qp->sq.lock); - spin_lock_init(&qp->rq.lock); - - ret = mthca_map_memfree(dev, qp); - if (ret) - return ret; - - ret = mthca_alloc_wqe_buf(dev, pd, qp); - if (ret) { - mthca_unmap_memfree(dev, qp); - return ret; - } - - mthca_adjust_qp_caps(dev, pd, qp); - - /* - * If this is a userspace QP, we're done now. The doorbells - * will be allocated and buffers will be initialized in - * userspace. - */ - if (pd->ibpd.uobject) - return 0; - - ret = mthca_alloc_memfree(dev, qp); - if (ret) { - mthca_free_wqe_buf(dev, qp); - mthca_unmap_memfree(dev, qp); - return ret; - } - - if (mthca_is_memfree(dev)) { - struct mthca_data_seg *scatter; - int size = (sizeof (struct mthca_next_seg) + - qp->rq.max_gs * sizeof (struct mthca_data_seg)) / 16; - - for (i = 0; i < qp->rq.max; ++i) { - next = get_recv_wqe(qp, i); - next->nda_op = cpu_to_be32(((i + 1) & (qp->rq.max - 1)) << - qp->rq.wqe_shift); - next->ee_nds = cpu_to_be32(size); - - for (scatter = (void *) (next + 1); - (void *) scatter < (void *) next + (1 << qp->rq.wqe_shift); - ++scatter) - scatter->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - } - - for (i = 0; i < qp->sq.max; ++i) { - next = get_send_wqe(qp, i); - next->nda_op = cpu_to_be32((((i + 1) & (qp->sq.max - 1)) << - qp->sq.wqe_shift) + - qp->send_wqe_offset); - } - } else { - for (i = 0; i < qp->rq.max; ++i) { - next = get_recv_wqe(qp, i); - next->nda_op = htonl((((i + 1) % qp->rq.max) << - qp->rq.wqe_shift) | 1); - } - - } - - qp->sq.last = get_send_wqe(qp, qp->sq.max - 1); - qp->rq.last = get_recv_wqe(qp, qp->rq.max - 1); - - return 0; -} - -static int mthca_set_qp_size(struct mthca_dev *dev, struct ib_qp_cap *cap, - struct mthca_pd *pd, struct mthca_qp *qp) -{ - int max_data_size = mthca_max_data_size(dev, qp, dev->limits.max_desc_sz); - u32 max_inline_data; - - /* Sanity check QP size before proceeding */ - if (cap->max_send_wr > dev->limits.max_wqes || - cap->max_recv_wr > dev->limits.max_wqes || - cap->max_send_sge > dev->limits.max_sg || - cap->max_recv_sge > dev->limits.max_sg) - return -EINVAL; - - if (pd->ibpd.uobject && - cap->max_inline_data > mthca_max_inline_data(pd, max_data_size)) - return -EINVAL; - - max_inline_data = pd->ibpd.uobject ? cap->max_inline_data : 0; - - /* - * For MLX transport we need 2 extra send gather entries: - * one for the header and one for the checksum at the end - */ - if (qp->transport == MLX && cap->max_send_sge + 2 > dev->limits.max_sg) - return -EINVAL; - - if (mthca_is_memfree(dev)) { - qp->rq.max = cap->max_recv_wr ? - roundup_pow_of_two(cap->max_recv_wr) : 0; - qp->sq.max = cap->max_send_wr ? - roundup_pow_of_two(cap->max_send_wr) : 0; - } else { - qp->rq.max = cap->max_recv_wr; - qp->sq.max = cap->max_send_wr; - } - - qp->rq.max_gs = cap->max_recv_sge; - qp->sq.max_gs = max_t(int, cap->max_send_sge, - ALIGN(max_inline_data + MTHCA_INLINE_HEADER_SIZE, - MTHCA_INLINE_CHUNK_SIZE) / - sizeof (struct mthca_data_seg)); - - return 0; -} - -int mthca_alloc_qp(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_cq *send_cq, - struct mthca_cq *recv_cq, - enum ib_qp_type type, - enum ib_sig_type send_policy, - struct ib_qp_cap *cap, - struct mthca_qp *qp) -{ - int err; - - switch (type) { - case IB_QPT_RC: qp->transport = RC; break; - case IB_QPT_UC: qp->transport = UC; break; - case IB_QPT_UD: qp->transport = UD; break; - default: return -EINVAL; - } - - err = mthca_set_qp_size(dev, cap, pd, qp); - if (err) - return err; - - qp->qpn = mthca_alloc(&dev->qp_table.alloc); - if (qp->qpn == -1) - return -ENOMEM; - - /* initialize port to zero for error-catching. */ - qp->port = 0; - - err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq, - send_policy, qp); - if (err) { - mthca_free(&dev->qp_table.alloc, qp->qpn); - return err; - } - - spin_lock_irq(&dev->qp_table.lock); - mthca_array_set(&dev->qp_table.qp, - qp->qpn & (dev->limits.num_qps - 1), qp); - spin_unlock_irq(&dev->qp_table.lock); - - return 0; -} - -static void mthca_lock_cqs(struct mthca_cq *send_cq, struct mthca_cq *recv_cq) -{ - if (send_cq == recv_cq) - spin_lock_irq(&send_cq->lock); - else if (send_cq->cqn < recv_cq->cqn) { - spin_lock_irq(&send_cq->lock); - spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); - } else { - spin_lock_irq(&recv_cq->lock); - spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); - } -} - -static void mthca_unlock_cqs(struct mthca_cq *send_cq, struct mthca_cq *recv_cq) -{ - if (send_cq == recv_cq) - spin_unlock_irq(&send_cq->lock); - else if (send_cq->cqn < recv_cq->cqn) { - spin_unlock(&recv_cq->lock); - spin_unlock_irq(&send_cq->lock); - } else { - spin_unlock(&send_cq->lock); - spin_unlock_irq(&recv_cq->lock); - } -} - -int mthca_alloc_sqp(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_cq *send_cq, - struct mthca_cq *recv_cq, - enum ib_sig_type send_policy, - struct ib_qp_cap *cap, - int qpn, - int port, - struct mthca_sqp *sqp) -{ - u32 mqpn = qpn * 2 + dev->qp_table.sqp_start + port - 1; - int err; - - sqp->qp.transport = MLX; - err = mthca_set_qp_size(dev, cap, pd, &sqp->qp); - if (err) - return err; - - sqp->header_buf_size = sqp->qp.sq.max * MTHCA_UD_HEADER_SIZE; - sqp->header_buf = dma_alloc_coherent(&dev->pdev->dev, sqp->header_buf_size, - &sqp->header_dma, GFP_KERNEL); - if (!sqp->header_buf) - return -ENOMEM; - - spin_lock_irq(&dev->qp_table.lock); - if (mthca_array_get(&dev->qp_table.qp, mqpn)) - err = -EBUSY; - else - mthca_array_set(&dev->qp_table.qp, mqpn, sqp); - spin_unlock_irq(&dev->qp_table.lock); - - if (err) - goto err_out; - - sqp->qp.port = port; - sqp->qp.qpn = mqpn; - sqp->qp.transport = MLX; - - err = mthca_alloc_qp_common(dev, pd, send_cq, recv_cq, - send_policy, &sqp->qp); - if (err) - goto err_out_free; - - atomic_inc(&pd->sqp_count); - - return 0; - - err_out_free: - /* - * Lock CQs here, so that CQ polling code can do QP lookup - * without taking a lock. - */ - mthca_lock_cqs(send_cq, recv_cq); - - spin_lock(&dev->qp_table.lock); - mthca_array_clear(&dev->qp_table.qp, mqpn); - spin_unlock(&dev->qp_table.lock); - - mthca_unlock_cqs(send_cq, recv_cq); - - err_out: - dma_free_coherent(&dev->pdev->dev, sqp->header_buf_size, - sqp->header_buf, sqp->header_dma); - - return err; -} - -static inline int get_qp_refcount(struct mthca_dev *dev, struct mthca_qp *qp) -{ - int c; - - spin_lock_irq(&dev->qp_table.lock); - c = qp->refcount; - spin_unlock_irq(&dev->qp_table.lock); - - return c; -} - -void mthca_free_qp(struct mthca_dev *dev, - struct mthca_qp *qp) -{ - u8 status; - struct mthca_cq *send_cq; - struct mthca_cq *recv_cq; - - send_cq = to_mcq(qp->ibqp.send_cq); - recv_cq = to_mcq(qp->ibqp.recv_cq); - - /* - * Lock CQs here, so that CQ polling code can do QP lookup - * without taking a lock. - */ - mthca_lock_cqs(send_cq, recv_cq); - - spin_lock(&dev->qp_table.lock); - mthca_array_clear(&dev->qp_table.qp, - qp->qpn & (dev->limits.num_qps - 1)); - --qp->refcount; - spin_unlock(&dev->qp_table.lock); - - mthca_unlock_cqs(send_cq, recv_cq); - - wait_event(qp->wait, !get_qp_refcount(dev, qp)); - - if (qp->state != IB_QPS_RESET) - mthca_MODIFY_QP(dev, qp->state, IB_QPS_RESET, qp->qpn, 0, - NULL, 0, &status); - - /* - * If this is a userspace QP, the buffers, MR, CQs and so on - * will be cleaned up in userspace, so all we have to do is - * unref the mem-free tables and free the QPN in our table. - */ - if (!qp->ibqp.uobject) { - mthca_cq_clean(dev, recv_cq, qp->qpn, - qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL); - if (send_cq != recv_cq) - mthca_cq_clean(dev, send_cq, qp->qpn, NULL); - - mthca_free_memfree(dev, qp); - mthca_free_wqe_buf(dev, qp); - } - - mthca_unmap_memfree(dev, qp); - - if (is_sqp(dev, qp)) { - atomic_dec(&(to_mpd(qp->ibqp.pd)->sqp_count)); - dma_free_coherent(&dev->pdev->dev, - to_msqp(qp)->header_buf_size, - to_msqp(qp)->header_buf, - to_msqp(qp)->header_dma); - } else - mthca_free(&dev->qp_table.alloc, qp->qpn); -} - -/* Create UD header for an MLX send and build a data segment for it */ -static int build_mlx_header(struct mthca_dev *dev, struct mthca_sqp *sqp, - int ind, struct ib_send_wr *wr, - struct mthca_mlx_seg *mlx, - struct mthca_data_seg *data) -{ - int header_size; - int err; - u16 pkey; - - ib_ud_header_init(256, /* assume a MAD */ - 1, 0, 0, - mthca_ah_grh_present(to_mah(wr->wr.ud.ah)), - 0, - &sqp->ud_header); - - err = mthca_read_ah(dev, to_mah(wr->wr.ud.ah), &sqp->ud_header); - if (err) - return err; - mlx->flags &= ~cpu_to_be32(MTHCA_NEXT_SOLICIT | 1); - mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MTHCA_MLX_VL15 : 0) | - (sqp->ud_header.lrh.destination_lid == - IB_LID_PERMISSIVE ? MTHCA_MLX_SLR : 0) | - (sqp->ud_header.lrh.service_level << 8)); - mlx->rlid = sqp->ud_header.lrh.destination_lid; - mlx->vcrc = 0; - - switch (wr->opcode) { - case IB_WR_SEND: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY; - sqp->ud_header.immediate_present = 0; - break; - case IB_WR_SEND_WITH_IMM: - sqp->ud_header.bth.opcode = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE; - sqp->ud_header.immediate_present = 1; - sqp->ud_header.immediate_data = wr->ex.imm_data; - break; - default: - return -EINVAL; - } - - sqp->ud_header.lrh.virtual_lane = !sqp->qp.ibqp.qp_num ? 15 : 0; - if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE) - sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE; - sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED); - if (!sqp->qp.ibqp.qp_num) - ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, - sqp->pkey_index, &pkey); - else - ib_get_cached_pkey(&dev->ib_dev, sqp->qp.port, - wr->wr.ud.pkey_index, &pkey); - sqp->ud_header.bth.pkey = cpu_to_be16(pkey); - sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn); - sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1)); - sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ? - sqp->qkey : wr->wr.ud.remote_qkey); - sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num); - - header_size = ib_ud_header_pack(&sqp->ud_header, - sqp->header_buf + - ind * MTHCA_UD_HEADER_SIZE); - - data->byte_count = cpu_to_be32(header_size); - data->lkey = cpu_to_be32(to_mpd(sqp->qp.ibqp.pd)->ntmr.ibmr.lkey); - data->addr = cpu_to_be64(sqp->header_dma + - ind * MTHCA_UD_HEADER_SIZE); - - return 0; -} - -static inline int mthca_wq_overflow(struct mthca_wq *wq, int nreq, - struct ib_cq *ib_cq) -{ - unsigned cur; - struct mthca_cq *cq; - - cur = wq->head - wq->tail; - if (likely(cur + nreq < wq->max)) - return 0; - - cq = to_mcq(ib_cq); - spin_lock(&cq->lock); - cur = wq->head - wq->tail; - spin_unlock(&cq->lock); - - return cur + nreq >= wq->max; -} - -static __always_inline void set_raddr_seg(struct mthca_raddr_seg *rseg, - u64 remote_addr, u32 rkey) -{ - rseg->raddr = cpu_to_be64(remote_addr); - rseg->rkey = cpu_to_be32(rkey); - rseg->reserved = 0; -} - -static __always_inline void set_atomic_seg(struct mthca_atomic_seg *aseg, - struct ib_send_wr *wr) -{ - if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) { - aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap); - aseg->compare = cpu_to_be64(wr->wr.atomic.compare_add); - } else { - aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add); - aseg->compare = 0; - } - -} - -static void set_tavor_ud_seg(struct mthca_tavor_ud_seg *useg, - struct ib_send_wr *wr) -{ - useg->lkey = cpu_to_be32(to_mah(wr->wr.ud.ah)->key); - useg->av_addr = cpu_to_be64(to_mah(wr->wr.ud.ah)->avdma); - useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); - useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); - -} - -static void set_arbel_ud_seg(struct mthca_arbel_ud_seg *useg, - struct ib_send_wr *wr) -{ - memcpy(useg->av, to_mah(wr->wr.ud.ah)->av, MTHCA_AV_SIZE); - useg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn); - useg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey); -} - -int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - void *wqe; - void *prev_wqe; - unsigned long flags; - int err = 0; - int nreq; - int i; - int size; - /* - * f0 and size0 are only used if nreq != 0, and they will - * always be initialized the first time through the main loop - * before nreq is incremented. So nreq cannot become non-zero - * without initializing f0 and size0, and they are in fact - * never used uninitialized. - */ - int uninitialized_var(size0); - u32 uninitialized_var(f0); - int ind; - u8 op0 = 0; - - spin_lock_irqsave(&qp->sq.lock, flags); - - /* XXX check that state is OK to post send */ - - ind = qp->sq.next_ind; - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - mthca_err(dev, "SQ %06x full (%u head, %u tail," - " %d max, %d nreq)\n", qp->qpn, - qp->sq.head, qp->sq.tail, - qp->sq.max, nreq); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe = get_send_wqe(qp, ind); - prev_wqe = qp->sq.last; - qp->sq.last = wqe; - - ((struct mthca_next_seg *) wqe)->nda_op = 0; - ((struct mthca_next_seg *) wqe)->ee_nds = 0; - ((struct mthca_next_seg *) wqe)->flags = - ((wr->send_flags & IB_SEND_SIGNALED) ? - cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0) | - ((wr->send_flags & IB_SEND_SOLICITED) ? - cpu_to_be32(MTHCA_NEXT_SOLICIT) : 0) | - cpu_to_be32(1); - if (wr->opcode == IB_WR_SEND_WITH_IMM || - wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) - ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; - - wqe += sizeof (struct mthca_next_seg); - size = sizeof (struct mthca_next_seg) / 16; - - switch (qp->transport) { - case RC: - switch (wr->opcode) { - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - set_raddr_seg(wqe, wr->wr.atomic.remote_addr, - wr->wr.atomic.rkey); - wqe += sizeof (struct mthca_raddr_seg); - - set_atomic_seg(wqe, wr); - wqe += sizeof (struct mthca_atomic_seg); - size += (sizeof (struct mthca_raddr_seg) + - sizeof (struct mthca_atomic_seg)) / 16; - break; - - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - case IB_WR_RDMA_READ: - set_raddr_seg(wqe, wr->wr.rdma.remote_addr, - wr->wr.rdma.rkey); - wqe += sizeof (struct mthca_raddr_seg); - size += sizeof (struct mthca_raddr_seg) / 16; - break; - - default: - /* No extra segments required for sends */ - break; - } - - break; - - case UC: - switch (wr->opcode) { - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - set_raddr_seg(wqe, wr->wr.rdma.remote_addr, - wr->wr.rdma.rkey); - wqe += sizeof (struct mthca_raddr_seg); - size += sizeof (struct mthca_raddr_seg) / 16; - break; - - default: - /* No extra segments required for sends */ - break; - } - - break; - - case UD: - set_tavor_ud_seg(wqe, wr); - wqe += sizeof (struct mthca_tavor_ud_seg); - size += sizeof (struct mthca_tavor_ud_seg) / 16; - break; - - case MLX: - err = build_mlx_header(dev, to_msqp(qp), ind, wr, - wqe - sizeof (struct mthca_next_seg), - wqe); - if (err) { - *bad_wr = wr; - goto out; - } - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - break; - } - - if (wr->num_sge > qp->sq.max_gs) { - mthca_err(dev, "too many gathers\n"); - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - } - - /* Add one more inline data segment for ICRC */ - if (qp->transport == MLX) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32((1U << 31) | 4); - ((u32 *) wqe)[1] = 0; - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - } - - qp->wrid[ind] = wr->wr_id; - - if (wr->opcode >= ARRAY_SIZE(mthca_opcode)) { - mthca_err(dev, "opcode invalid\n"); - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - ((struct mthca_next_seg *) prev_wqe)->nda_op = - cpu_to_be32(((ind << qp->sq.wqe_shift) + - qp->send_wqe_offset) | - mthca_opcode[wr->opcode]); - wmb(); - ((struct mthca_next_seg *) prev_wqe)->ee_nds = - cpu_to_be32((nreq ? 0 : MTHCA_NEXT_DBD) | size | - ((wr->send_flags & IB_SEND_FENCE) ? - MTHCA_NEXT_FENCE : 0)); - - if (!nreq) { - size0 = size; - op0 = mthca_opcode[wr->opcode]; - f0 = wr->send_flags & IB_SEND_FENCE ? - MTHCA_SEND_DOORBELL_FENCE : 0; - } - - ++ind; - if (unlikely(ind >= qp->sq.max)) - ind -= qp->sq.max; - } - -out: - if (likely(nreq)) { - wmb(); - - mthca_write64(((qp->sq.next_ind << qp->sq.wqe_shift) + - qp->send_wqe_offset) | f0 | op0, - (qp->qpn << 8) | size0, - dev->kar + MTHCA_SEND_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - /* - * Make sure doorbells don't leak out of SQ spinlock - * and reach the HCA out of order: - */ - mmiowb(); - } - - qp->sq.next_ind = ind; - qp->sq.head += nreq; - - spin_unlock_irqrestore(&qp->sq.lock, flags); - return err; -} - -int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - unsigned long flags; - int err = 0; - int nreq; - int i; - int size; - /* - * size0 is only used if nreq != 0, and it will always be - * initialized the first time through the main loop before - * nreq is incremented. So nreq cannot become non-zero - * without initializing size0, and it is in fact never used - * uninitialized. - */ - int uninitialized_var(size0); - int ind; - void *wqe; - void *prev_wqe; - - spin_lock_irqsave(&qp->rq.lock, flags); - - /* XXX check that state is OK to post receive */ - - ind = qp->rq.next_ind; - - for (nreq = 0; wr; wr = wr->next) { - if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { - mthca_err(dev, "RQ %06x full (%u head, %u tail," - " %d max, %d nreq)\n", qp->qpn, - qp->rq.head, qp->rq.tail, - qp->rq.max, nreq); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe = get_recv_wqe(qp, ind); - prev_wqe = qp->rq.last; - qp->rq.last = wqe; - - ((struct mthca_next_seg *) wqe)->ee_nds = - cpu_to_be32(MTHCA_NEXT_DBD); - ((struct mthca_next_seg *) wqe)->flags = 0; - - wqe += sizeof (struct mthca_next_seg); - size = sizeof (struct mthca_next_seg) / 16; - - if (unlikely(wr->num_sge > qp->rq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - } - - qp->wrid[ind + qp->sq.max] = wr->wr_id; - - ((struct mthca_next_seg *) prev_wqe)->ee_nds = - cpu_to_be32(MTHCA_NEXT_DBD | size); - - if (!nreq) - size0 = size; - - ++ind; - if (unlikely(ind >= qp->rq.max)) - ind -= qp->rq.max; - - ++nreq; - if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) { - nreq = 0; - - wmb(); - - mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, - qp->qpn << 8, dev->kar + MTHCA_RECEIVE_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - - qp->rq.next_ind = ind; - qp->rq.head += MTHCA_TAVOR_MAX_WQES_PER_RECV_DB; - } - } - -out: - if (likely(nreq)) { - wmb(); - - mthca_write64((qp->rq.next_ind << qp->rq.wqe_shift) | size0, - qp->qpn << 8 | nreq, dev->kar + MTHCA_RECEIVE_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - } - - qp->rq.next_ind = ind; - qp->rq.head += nreq; - - /* - * Make sure doorbells don't leak out of RQ spinlock and reach - * the HCA out of order: - */ - mmiowb(); - - spin_unlock_irqrestore(&qp->rq.lock, flags); - return err; -} - -int mthca_arbel_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, - struct ib_send_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - u32 dbhi; - void *wqe; - void *prev_wqe; - unsigned long flags; - int err = 0; - int nreq; - int i; - int size; - /* - * f0 and size0 are only used if nreq != 0, and they will - * always be initialized the first time through the main loop - * before nreq is incremented. So nreq cannot become non-zero - * without initializing f0 and size0, and they are in fact - * never used uninitialized. - */ - int uninitialized_var(size0); - u32 uninitialized_var(f0); - int ind; - u8 op0 = 0; - - spin_lock_irqsave(&qp->sq.lock, flags); - - /* XXX check that state is OK to post send */ - - ind = qp->sq.head & (qp->sq.max - 1); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (unlikely(nreq == MTHCA_ARBEL_MAX_WQES_PER_SEND_DB)) { - nreq = 0; - - dbhi = (MTHCA_ARBEL_MAX_WQES_PER_SEND_DB << 24) | - ((qp->sq.head & 0xffff) << 8) | f0 | op0; - - qp->sq.head += MTHCA_ARBEL_MAX_WQES_PER_SEND_DB; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - *qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff); - - /* - * Make sure doorbell record is written before we - * write MMIO send doorbell. - */ - wmb(); - - mthca_write64(dbhi, (qp->qpn << 8) | size0, - dev->kar + MTHCA_SEND_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - } - - if (mthca_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) { - mthca_err(dev, "SQ %06x full (%u head, %u tail," - " %d max, %d nreq)\n", qp->qpn, - qp->sq.head, qp->sq.tail, - qp->sq.max, nreq); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe = get_send_wqe(qp, ind); - prev_wqe = qp->sq.last; - qp->sq.last = wqe; - - ((struct mthca_next_seg *) wqe)->flags = - ((wr->send_flags & IB_SEND_SIGNALED) ? - cpu_to_be32(MTHCA_NEXT_CQ_UPDATE) : 0) | - ((wr->send_flags & IB_SEND_SOLICITED) ? - cpu_to_be32(MTHCA_NEXT_SOLICIT) : 0) | - ((wr->send_flags & IB_SEND_IP_CSUM) ? - cpu_to_be32(MTHCA_NEXT_IP_CSUM | MTHCA_NEXT_TCP_UDP_CSUM) : 0) | - cpu_to_be32(1); - if (wr->opcode == IB_WR_SEND_WITH_IMM || - wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) - ((struct mthca_next_seg *) wqe)->imm = wr->ex.imm_data; - - wqe += sizeof (struct mthca_next_seg); - size = sizeof (struct mthca_next_seg) / 16; - - switch (qp->transport) { - case RC: - switch (wr->opcode) { - case IB_WR_ATOMIC_CMP_AND_SWP: - case IB_WR_ATOMIC_FETCH_AND_ADD: - set_raddr_seg(wqe, wr->wr.atomic.remote_addr, - wr->wr.atomic.rkey); - wqe += sizeof (struct mthca_raddr_seg); - - set_atomic_seg(wqe, wr); - wqe += sizeof (struct mthca_atomic_seg); - size += (sizeof (struct mthca_raddr_seg) + - sizeof (struct mthca_atomic_seg)) / 16; - break; - - case IB_WR_RDMA_READ: - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - set_raddr_seg(wqe, wr->wr.rdma.remote_addr, - wr->wr.rdma.rkey); - wqe += sizeof (struct mthca_raddr_seg); - size += sizeof (struct mthca_raddr_seg) / 16; - break; - - default: - /* No extra segments required for sends */ - break; - } - - break; - - case UC: - switch (wr->opcode) { - case IB_WR_RDMA_WRITE: - case IB_WR_RDMA_WRITE_WITH_IMM: - set_raddr_seg(wqe, wr->wr.rdma.remote_addr, - wr->wr.rdma.rkey); - wqe += sizeof (struct mthca_raddr_seg); - size += sizeof (struct mthca_raddr_seg) / 16; - break; - - default: - /* No extra segments required for sends */ - break; - } - - break; - - case UD: - set_arbel_ud_seg(wqe, wr); - wqe += sizeof (struct mthca_arbel_ud_seg); - size += sizeof (struct mthca_arbel_ud_seg) / 16; - break; - - case MLX: - err = build_mlx_header(dev, to_msqp(qp), ind, wr, - wqe - sizeof (struct mthca_next_seg), - wqe); - if (err) { - *bad_wr = wr; - goto out; - } - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - break; - } - - if (wr->num_sge > qp->sq.max_gs) { - mthca_err(dev, "too many gathers\n"); - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - } - - /* Add one more inline data segment for ICRC */ - if (qp->transport == MLX) { - ((struct mthca_data_seg *) wqe)->byte_count = - cpu_to_be32((1U << 31) | 4); - ((u32 *) wqe)[1] = 0; - wqe += sizeof (struct mthca_data_seg); - size += sizeof (struct mthca_data_seg) / 16; - } - - qp->wrid[ind] = wr->wr_id; - - if (wr->opcode >= ARRAY_SIZE(mthca_opcode)) { - mthca_err(dev, "opcode invalid\n"); - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - ((struct mthca_next_seg *) prev_wqe)->nda_op = - cpu_to_be32(((ind << qp->sq.wqe_shift) + - qp->send_wqe_offset) | - mthca_opcode[wr->opcode]); - wmb(); - ((struct mthca_next_seg *) prev_wqe)->ee_nds = - cpu_to_be32(MTHCA_NEXT_DBD | size | - ((wr->send_flags & IB_SEND_FENCE) ? - MTHCA_NEXT_FENCE : 0)); - - if (!nreq) { - size0 = size; - op0 = mthca_opcode[wr->opcode]; - f0 = wr->send_flags & IB_SEND_FENCE ? - MTHCA_SEND_DOORBELL_FENCE : 0; - } - - ++ind; - if (unlikely(ind >= qp->sq.max)) - ind -= qp->sq.max; - } - -out: - if (likely(nreq)) { - dbhi = (nreq << 24) | ((qp->sq.head & 0xffff) << 8) | f0 | op0; - - qp->sq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - *qp->sq.db = cpu_to_be32(qp->sq.head & 0xffff); - - /* - * Make sure doorbell record is written before we - * write MMIO send doorbell. - */ - wmb(); - - mthca_write64(dbhi, (qp->qpn << 8) | size0, dev->kar + MTHCA_SEND_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - } - - /* - * Make sure doorbells don't leak out of SQ spinlock and reach - * the HCA out of order: - */ - mmiowb(); - - spin_unlock_irqrestore(&qp->sq.lock, flags); - return err; -} - -int mthca_arbel_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibqp->device); - struct mthca_qp *qp = to_mqp(ibqp); - unsigned long flags; - int err = 0; - int nreq; - int ind; - int i; - void *wqe; - - spin_lock_irqsave(&qp->rq.lock, flags); - - /* XXX check that state is OK to post receive */ - - ind = qp->rq.head & (qp->rq.max - 1); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - if (mthca_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) { - mthca_err(dev, "RQ %06x full (%u head, %u tail," - " %d max, %d nreq)\n", qp->qpn, - qp->rq.head, qp->rq.tail, - qp->rq.max, nreq); - err = -ENOMEM; - *bad_wr = wr; - goto out; - } - - wqe = get_recv_wqe(qp, ind); - - ((struct mthca_next_seg *) wqe)->flags = 0; - - wqe += sizeof (struct mthca_next_seg); - - if (unlikely(wr->num_sge > qp->rq.max_gs)) { - err = -EINVAL; - *bad_wr = wr; - goto out; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - } - - if (i < qp->rq.max_gs) - mthca_set_data_seg_inval(wqe); - - qp->wrid[ind + qp->sq.max] = wr->wr_id; - - ++ind; - if (unlikely(ind >= qp->rq.max)) - ind -= qp->rq.max; - } -out: - if (likely(nreq)) { - qp->rq.head += nreq; - - /* - * Make sure that descriptors are written before - * doorbell record. - */ - wmb(); - *qp->rq.db = cpu_to_be32(qp->rq.head & 0xffff); - } - - spin_unlock_irqrestore(&qp->rq.lock, flags); - return err; -} - -void mthca_free_err_wqe(struct mthca_dev *dev, struct mthca_qp *qp, int is_send, - int index, int *dbd, __be32 *new_wqe) -{ - struct mthca_next_seg *next; - - /* - * For SRQs, all receive WQEs generate a CQE, so we're always - * at the end of the doorbell chain. - */ - if (qp->ibqp.srq && !is_send) { - *new_wqe = 0; - return; - } - - if (is_send) - next = get_send_wqe(qp, index); - else - next = get_recv_wqe(qp, index); - - *dbd = !!(next->ee_nds & cpu_to_be32(MTHCA_NEXT_DBD)); - if (next->ee_nds & cpu_to_be32(0x3f)) - *new_wqe = (next->nda_op & cpu_to_be32(~0x3f)) | - (next->ee_nds & cpu_to_be32(0x3f)); - else - *new_wqe = 0; -} - -int mthca_init_qp_table(struct mthca_dev *dev) -{ - int err; - u8 status; - int i; - - spin_lock_init(&dev->qp_table.lock); - - /* - * We reserve 2 extra QPs per port for the special QPs. The - * special QP for port 1 has to be even, so round up. - */ - dev->qp_table.sqp_start = (dev->limits.reserved_qps + 1) & ~1UL; - err = mthca_alloc_init(&dev->qp_table.alloc, - dev->limits.num_qps, - (1 << 24) - 1, - dev->qp_table.sqp_start + - MTHCA_MAX_PORTS * 2); - if (err) - return err; - - err = mthca_array_init(&dev->qp_table.qp, - dev->limits.num_qps); - if (err) { - mthca_alloc_cleanup(&dev->qp_table.alloc); - return err; - } - - for (i = 0; i < 2; ++i) { - err = mthca_CONF_SPECIAL_QP(dev, i ? IB_QPT_GSI : IB_QPT_SMI, - dev->qp_table.sqp_start + i * 2, - &status); - if (err) - goto err_out; - if (status) { - mthca_warn(dev, "CONF_SPECIAL_QP returned " - "status %02x, aborting.\n", - status); - err = -EINVAL; - goto err_out; - } - } - return 0; - - err_out: - for (i = 0; i < 2; ++i) - mthca_CONF_SPECIAL_QP(dev, i, 0, &status); - - mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); - mthca_alloc_cleanup(&dev->qp_table.alloc); - - return err; -} - -void mthca_cleanup_qp_table(struct mthca_dev *dev) -{ - int i; - u8 status; - - for (i = 0; i < 2; ++i) - mthca_CONF_SPECIAL_QP(dev, i, 0, &status); - - mthca_array_cleanup(&dev->qp_table.qp, dev->limits.num_qps); - mthca_alloc_cleanup(&dev->qp_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c deleted file mode 100644 index f571bd6d1ca2..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_reset.c +++ /dev/null @@ -1,303 +0,0 @@ -/* - * Copyright (c) 2004 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" - -int mthca_reset(struct mthca_dev *mdev) -{ - int i; - int err = 0; - u32 *hca_header = NULL; -#ifdef __linux__ - u32 *bridge_header = NULL; -#endif - struct pci_dev *bridge = NULL; -#ifdef __linux__ - int bridge_pcix_cap = 0; -#endif - int hca_pcie_cap = 0; - int hca_pcix_cap = 0; - - u16 devctl; - u16 linkctl; - -#define MTHCA_RESET_OFFSET 0xf0010 -#define MTHCA_RESET_VALUE swab32(1) - - /* - * Reset the chip. This is somewhat ugly because we have to - * save off the PCI header before reset and then restore it - * after the chip reboots. We skip config space offsets 22 - * and 23 since those have a special meaning. - * - * To make matters worse, for Tavor (PCI-X HCA) we have to - * find the associated bridge device and save off its PCI - * header as well. - */ - - if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE)) { - /* Look for the bridge -- its device ID will be 2 more - than HCA's device ID. */ -#ifdef __linux__ - while ((bridge = pci_get_device(mdev->pdev->vendor, - mdev->pdev->device + 2, - bridge)) != NULL) { - if (bridge->hdr_type == PCI_HEADER_TYPE_BRIDGE && - bridge->subordinate == mdev->pdev->bus) { - mthca_dbg(mdev, "Found bridge: %s\n", - pci_name(bridge)); - break; - } - } - - if (!bridge) { - /* - * Didn't find a bridge for a Tavor device -- - * assume we're in no-bridge mode and hope for - * the best. - */ - mthca_warn(mdev, "No bridge found for %s\n", - pci_name(mdev->pdev)); - } -#else - mthca_warn(mdev, "Reset on PCI-X is not supported.\n"); - goto out; - -#endif - } - - /* For Arbel do we need to save off the full 4K PCI Express header?? */ - hca_header = kmalloc(256, GFP_KERNEL); - if (!hca_header) { - err = -ENOMEM; - mthca_err(mdev, "Couldn't allocate memory to save HCA " - "PCI header, aborting.\n"); - goto out; - } - - for (i = 0; i < 64; ++i) { - if (i == 22 || i == 23) - continue; - if (pci_read_config_dword(mdev->pdev, i * 4, hca_header + i)) { - err = -ENODEV; - mthca_err(mdev, "Couldn't save HCA " - "PCI header, aborting.\n"); - goto out; - } - } - - hca_pcix_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX); - hca_pcie_cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP); - -#ifdef __linux__ - if (bridge) { - bridge_header = kmalloc(256, GFP_KERNEL); - if (!bridge_header) { - err = -ENOMEM; - mthca_err(mdev, "Couldn't allocate memory to save HCA " - "bridge PCI header, aborting.\n"); - goto out; - } - - for (i = 0; i < 64; ++i) { - if (i == 22 || i == 23) - continue; - if (pci_read_config_dword(bridge, i * 4, bridge_header + i)) { - err = -ENODEV; - mthca_err(mdev, "Couldn't save HCA bridge " - "PCI header, aborting.\n"); - goto out; - } - } - bridge_pcix_cap = pci_find_capability(bridge, PCI_CAP_ID_PCIX); - if (!bridge_pcix_cap) { - err = -ENODEV; - mthca_err(mdev, "Couldn't locate HCA bridge " - "PCI-X capability, aborting.\n"); - goto out; - } - } -#endif - - /* actually hit reset */ - { - void __iomem *reset = ioremap(pci_resource_start(mdev->pdev, 0) + - MTHCA_RESET_OFFSET, 4); - - if (!reset) { - err = -ENOMEM; - mthca_err(mdev, "Couldn't map HCA reset register, " - "aborting.\n"); - goto out; - } - - writel(MTHCA_RESET_VALUE, reset); - iounmap(reset); - } - - /* Docs say to wait one second before accessing device */ - msleep(1000); - - /* Now wait for PCI device to start responding again */ - { - u32 v; - int c = 0; - - for (c = 0; c < 100; ++c) { - if (pci_read_config_dword(bridge ? bridge : mdev->pdev, 0, &v)) { - err = -ENODEV; - mthca_err(mdev, "Couldn't access HCA after reset, " - "aborting.\n"); - goto out; - } - - if (v != 0xffffffff) - goto good; - - msleep(100); - } - - err = -ENODEV; - mthca_err(mdev, "PCI device did not come back after reset, " - "aborting.\n"); - goto out; - } - -good: -#ifdef __linux__ - /* Now restore the PCI headers */ - if (bridge) { - if (pci_write_config_dword(bridge, bridge_pcix_cap + 0x8, - bridge_header[(bridge_pcix_cap + 0x8) / 4])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA bridge Upstream " - "split transaction control, aborting.\n"); - goto out; - } - if (pci_write_config_dword(bridge, bridge_pcix_cap + 0xc, - bridge_header[(bridge_pcix_cap + 0xc) / 4])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA bridge Downstream " - "split transaction control, aborting.\n"); - goto out; - } - /* - * Bridge control register is at 0x3e, so we'll - * naturally restore it last in this loop. - */ - for (i = 0; i < 16; ++i) { - if (i * 4 == PCI_COMMAND) - continue; - - if (pci_write_config_dword(bridge, i * 4, bridge_header[i])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA bridge reg %x, " - "aborting.\n", i); - goto out; - } - } - - if (pci_write_config_dword(bridge, PCI_COMMAND, - bridge_header[PCI_COMMAND / 4])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA bridge COMMAND, " - "aborting.\n"); - goto out; - } - } -#endif - - if (hca_pcix_cap) { - if (pci_write_config_dword(mdev->pdev, hca_pcix_cap, - hca_header[hca_pcix_cap / 4])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA PCI-X " - "command register, aborting.\n"); - goto out; - } - } - - if (hca_pcie_cap) { - devctl = hca_header[(hca_pcie_cap + PCI_EXP_DEVCTL) / 4]; - if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_DEVCTL, - devctl)) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA PCI Express " - "Device Control register, aborting.\n"); - goto out; - } - linkctl = hca_header[(hca_pcie_cap + PCI_EXP_LNKCTL) / 4]; - if (pci_write_config_word(mdev->pdev, hca_pcie_cap + PCI_EXP_LNKCTL, - linkctl)) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA PCI Express " - "Link control register, aborting.\n"); - goto out; - } - } - - for (i = 0; i < 16; ++i) { - if (i * 4 == PCI_COMMAND) - continue; - - if (pci_write_config_dword(mdev->pdev, i * 4, hca_header[i])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA reg %x, " - "aborting.\n", i); - goto out; - } - } - - if (pci_write_config_dword(mdev->pdev, PCI_COMMAND, - hca_header[PCI_COMMAND / 4])) { - err = -ENODEV; - mthca_err(mdev, "Couldn't restore HCA COMMAND, " - "aborting.\n"); - goto out; - } - -out: -#ifdef __linux__ - if (bridge) - pci_dev_put(bridge); - kfree(bridge_header); -#endif - kfree(hca_header); - - return err; -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_srq.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_srq.c deleted file mode 100644 index 4fabe62aab8a..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_srq.c +++ /dev/null @@ -1,715 +0,0 @@ -/* - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include -#include -#include - -#include - -#include "mthca_dev.h" -#include "mthca_cmd.h" -#include "mthca_memfree.h" -#include "mthca_wqe.h" - -enum { - MTHCA_MAX_DIRECT_SRQ_SIZE = 4 * PAGE_SIZE -}; - -struct mthca_tavor_srq_context { - __be64 wqe_base_ds; /* low 6 bits is descriptor size */ - __be32 state_pd; - __be32 lkey; - __be32 uar; - __be16 limit_watermark; - __be16 wqe_cnt; - u32 reserved[2]; -}; - -struct mthca_arbel_srq_context { - __be32 state_logsize_srqn; - __be32 lkey; - __be32 db_index; - __be32 logstride_usrpage; - __be64 wqe_base; - __be32 eq_pd; - __be16 limit_watermark; - __be16 wqe_cnt; - u16 reserved1; - __be16 wqe_counter; - u32 reserved2[3]; -}; - -static void *get_wqe(struct mthca_srq *srq, int n) -{ - if (srq->is_direct) - return srq->queue.direct.buf + (n << srq->wqe_shift); - else - return srq->queue.page_list[(n << srq->wqe_shift) >> PAGE_SHIFT].buf + - ((n << srq->wqe_shift) & (PAGE_SIZE - 1)); -} - -/* - * Return a pointer to the location within a WQE that we're using as a - * link when the WQE is in the free list. We use the imm field - * because in the Tavor case, posting a WQE may overwrite the next - * segment of the previous WQE, but a receive WQE will never touch the - * imm field. This avoids corrupting our free list if the previous - * WQE has already completed and been put on the free list when we - * post the next WQE. - */ -static inline int *wqe_to_link(void *wqe) -{ - return (int *) (wqe + offsetof(struct mthca_next_seg, imm)); -} - -static void mthca_tavor_init_srq_context(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_srq *srq, - struct mthca_tavor_srq_context *context) -{ - memset(context, 0, sizeof *context); - - context->wqe_base_ds = cpu_to_be64(1 << (srq->wqe_shift - 4)); - context->state_pd = cpu_to_be32(pd->pd_num); - context->lkey = cpu_to_be32(srq->mr.ibmr.lkey); - - if (pd->ibpd.uobject) - context->uar = - cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index); - else - context->uar = cpu_to_be32(dev->driver_uar.index); -} - -static void mthca_arbel_init_srq_context(struct mthca_dev *dev, - struct mthca_pd *pd, - struct mthca_srq *srq, - struct mthca_arbel_srq_context *context) -{ - int logsize, max; - - memset(context, 0, sizeof *context); - - /* - * Put max in a temporary variable to work around gcc bug - * triggered by ilog2() on sparc64. - */ - max = srq->max; - logsize = ilog2(max); - context->state_logsize_srqn = cpu_to_be32(logsize << 24 | srq->srqn); - context->lkey = cpu_to_be32(srq->mr.ibmr.lkey); - context->db_index = cpu_to_be32(srq->db_index); - context->logstride_usrpage = cpu_to_be32((srq->wqe_shift - 4) << 29); - if (pd->ibpd.uobject) - context->logstride_usrpage |= - cpu_to_be32(to_mucontext(pd->ibpd.uobject->context)->uar.index); - else - context->logstride_usrpage |= cpu_to_be32(dev->driver_uar.index); - context->eq_pd = cpu_to_be32(MTHCA_EQ_ASYNC << 24 | pd->pd_num); -} - -static void mthca_free_srq_buf(struct mthca_dev *dev, struct mthca_srq *srq) -{ - mthca_buf_free(dev, srq->max << srq->wqe_shift, &srq->queue, - srq->is_direct, &srq->mr); - kfree(srq->wrid); -} - -static int mthca_alloc_srq_buf(struct mthca_dev *dev, struct mthca_pd *pd, - struct mthca_srq *srq) -{ - struct mthca_data_seg *scatter; - void *wqe; - int err; - int i; - - if (pd->ibpd.uobject) - return 0; - - srq->wrid = kmalloc(srq->max * sizeof (u64), GFP_KERNEL); - if (!srq->wrid) - return -ENOMEM; - - err = mthca_buf_alloc(dev, srq->max << srq->wqe_shift, - MTHCA_MAX_DIRECT_SRQ_SIZE, - &srq->queue, &srq->is_direct, pd, 1, &srq->mr); - if (err) { - kfree(srq->wrid); - return err; - } - - /* - * Now initialize the SRQ buffer so that all of the WQEs are - * linked into the list of free WQEs. In addition, set the - * scatter list L_Keys to the sentry value of 0x100. - */ - for (i = 0; i < srq->max; ++i) { - struct mthca_next_seg *next; - - next = wqe = get_wqe(srq, i); - - if (i < srq->max - 1) { - *wqe_to_link(wqe) = i + 1; - next->nda_op = htonl(((i + 1) << srq->wqe_shift) | 1); - } else { - *wqe_to_link(wqe) = -1; - next->nda_op = 0; - } - - for (scatter = wqe + sizeof (struct mthca_next_seg); - (void *) scatter < wqe + (1 << srq->wqe_shift); - ++scatter) - scatter->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - } - - srq->last = get_wqe(srq, srq->max - 1); - - return 0; -} - -int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd, - struct ib_srq_attr *attr, struct mthca_srq *srq) -{ - struct mthca_mailbox *mailbox; - u8 status; - int ds; - int err; - - /* Sanity check SRQ size before proceeding */ - if (attr->max_wr > dev->limits.max_srq_wqes || - attr->max_sge > dev->limits.max_srq_sge) - return -EINVAL; - - srq->max = attr->max_wr; - srq->max_gs = attr->max_sge; - srq->counter = 0; - - if (mthca_is_memfree(dev)) - srq->max = roundup_pow_of_two(srq->max + 1); - else - srq->max = srq->max + 1; - - ds = max(64UL, - roundup_pow_of_two(sizeof (struct mthca_next_seg) + - srq->max_gs * sizeof (struct mthca_data_seg))); - - if (!mthca_is_memfree(dev) && (ds > dev->limits.max_desc_sz)) - return -EINVAL; - - srq->wqe_shift = ilog2(ds); - - srq->srqn = mthca_alloc(&dev->srq_table.alloc); - if (srq->srqn == -1) - return -ENOMEM; - - if (mthca_is_memfree(dev)) { - err = mthca_table_get(dev, dev->srq_table.table, srq->srqn); - if (err) - goto err_out; - - if (!pd->ibpd.uobject) { - srq->db_index = mthca_alloc_db(dev, MTHCA_DB_TYPE_SRQ, - srq->srqn, &srq->db); - if (srq->db_index < 0) { - err = -ENOMEM; - goto err_out_icm; - } - } - } - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - err = PTR_ERR(mailbox); - goto err_out_db; - } - - err = mthca_alloc_srq_buf(dev, pd, srq); - if (err) - goto err_out_mailbox; - - spin_lock_init(&srq->lock); - srq->refcount = 1; - init_waitqueue_head(&srq->wait); - mutex_init(&srq->mutex); - - if (mthca_is_memfree(dev)) - mthca_arbel_init_srq_context(dev, pd, srq, mailbox->buf); - else - mthca_tavor_init_srq_context(dev, pd, srq, mailbox->buf); - - err = mthca_SW2HW_SRQ(dev, mailbox, srq->srqn, &status); - - if (err) { - mthca_warn(dev, "SW2HW_SRQ failed (%d)\n", err); - goto err_out_free_buf; - } - if (status) { - mthca_warn(dev, "SW2HW_SRQ returned status 0x%02x\n", - status); - err = -EINVAL; - goto err_out_free_buf; - } - - spin_lock_irq(&dev->srq_table.lock); - if (mthca_array_set(&dev->srq_table.srq, - srq->srqn & (dev->limits.num_srqs - 1), - srq)) { - spin_unlock_irq(&dev->srq_table.lock); - goto err_out_free_srq; - } - spin_unlock_irq(&dev->srq_table.lock); - - mthca_free_mailbox(dev, mailbox); - - srq->first_free = 0; - srq->last_free = srq->max - 1; - - attr->max_wr = srq->max - 1; - attr->max_sge = srq->max_gs; - - return 0; - -err_out_free_srq: - err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn, &status); - if (err) - mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_SRQ returned status 0x%02x\n", status); - -err_out_free_buf: - if (!pd->ibpd.uobject) - mthca_free_srq_buf(dev, srq); - -err_out_mailbox: - mthca_free_mailbox(dev, mailbox); - -err_out_db: - if (!pd->ibpd.uobject && mthca_is_memfree(dev)) - mthca_free_db(dev, MTHCA_DB_TYPE_SRQ, srq->db_index); - -err_out_icm: - mthca_table_put(dev, dev->srq_table.table, srq->srqn); - -err_out: - mthca_free(&dev->srq_table.alloc, srq->srqn); - - return err; -} - -static inline int get_srq_refcount(struct mthca_dev *dev, struct mthca_srq *srq) -{ - int c; - - spin_lock_irq(&dev->srq_table.lock); - c = srq->refcount; - spin_unlock_irq(&dev->srq_table.lock); - - return c; -} - -void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq) -{ - struct mthca_mailbox *mailbox; - int err; - u8 status; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) { - mthca_warn(dev, "No memory for mailbox to free SRQ.\n"); - return; - } - - err = mthca_HW2SW_SRQ(dev, mailbox, srq->srqn, &status); - if (err) - mthca_warn(dev, "HW2SW_SRQ failed (%d)\n", err); - else if (status) - mthca_warn(dev, "HW2SW_SRQ returned status 0x%02x\n", status); - - spin_lock_irq(&dev->srq_table.lock); - mthca_array_clear(&dev->srq_table.srq, - srq->srqn & (dev->limits.num_srqs - 1)); - --srq->refcount; - spin_unlock_irq(&dev->srq_table.lock); - - wait_event(srq->wait, !get_srq_refcount(dev, srq)); - - if (!srq->ibsrq.uobject) { - mthca_free_srq_buf(dev, srq); - if (mthca_is_memfree(dev)) - mthca_free_db(dev, MTHCA_DB_TYPE_SRQ, srq->db_index); - } - - mthca_table_put(dev, dev->srq_table.table, srq->srqn); - mthca_free(&dev->srq_table.alloc, srq->srqn); - mthca_free_mailbox(dev, mailbox); -} - -int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr, - enum ib_srq_attr_mask attr_mask, struct ib_udata *udata) -{ - struct mthca_dev *dev = to_mdev(ibsrq->device); - struct mthca_srq *srq = to_msrq(ibsrq); - int ret; - u8 status; - - /* We don't support resizing SRQs (yet?) */ - if (attr_mask & IB_SRQ_MAX_WR) - return -EINVAL; - - if (attr_mask & IB_SRQ_LIMIT) { - u32 max_wr = mthca_is_memfree(dev) ? srq->max - 1 : srq->max; - if (attr->srq_limit > max_wr) - return -EINVAL; - - mutex_lock(&srq->mutex); - ret = mthca_ARM_SRQ(dev, srq->srqn, attr->srq_limit, &status); - mutex_unlock(&srq->mutex); - - if (ret) - return ret; - if (status) - return -EINVAL; - } - - return 0; -} - -int mthca_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr) -{ - struct mthca_dev *dev = to_mdev(ibsrq->device); - struct mthca_srq *srq = to_msrq(ibsrq); - struct mthca_mailbox *mailbox; - struct mthca_arbel_srq_context *arbel_ctx; - struct mthca_tavor_srq_context *tavor_ctx; - u8 status; - int err; - - mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); - if (IS_ERR(mailbox)) - return PTR_ERR(mailbox); - - err = mthca_QUERY_SRQ(dev, srq->srqn, mailbox, &status); - if (err) - goto out; - - if (mthca_is_memfree(dev)) { - arbel_ctx = mailbox->buf; - srq_attr->srq_limit = be16_to_cpu(arbel_ctx->limit_watermark); - } else { - tavor_ctx = mailbox->buf; - srq_attr->srq_limit = be16_to_cpu(tavor_ctx->limit_watermark); - } - - srq_attr->max_wr = srq->max - 1; - srq_attr->max_sge = srq->max_gs; - -out: - mthca_free_mailbox(dev, mailbox); - - return err; -} - -void mthca_srq_event(struct mthca_dev *dev, u32 srqn, - enum ib_event_type event_type) -{ - struct mthca_srq *srq; - struct ib_event event; - - spin_lock(&dev->srq_table.lock); - srq = mthca_array_get(&dev->srq_table.srq, srqn & (dev->limits.num_srqs - 1)); - if (srq) - ++srq->refcount; - spin_unlock(&dev->srq_table.lock); - - if (!srq) { - mthca_warn(dev, "Async event for bogus SRQ %08x\n", srqn); - return; - } - - if (!srq->ibsrq.event_handler) - goto out; - - event.device = &dev->ib_dev; - event.event = event_type; - event.element.srq = &srq->ibsrq; - srq->ibsrq.event_handler(&event, srq->ibsrq.srq_context); - -out: - spin_lock(&dev->srq_table.lock); - if (!--srq->refcount) - wake_up(&srq->wait); - spin_unlock(&dev->srq_table.lock); -} - -/* - * This function must be called with IRQs disabled. - */ -void mthca_free_srq_wqe(struct mthca_srq *srq, u32 wqe_addr) -{ - int ind; - struct mthca_next_seg *last_free; - - ind = wqe_addr >> srq->wqe_shift; - - spin_lock(&srq->lock); - - last_free = get_wqe(srq, srq->last_free); - *wqe_to_link(last_free) = ind; - last_free->nda_op = htonl((ind << srq->wqe_shift) | 1); - *wqe_to_link(get_wqe(srq, ind)) = -1; - srq->last_free = ind; - - spin_unlock(&srq->lock); -} - -int mthca_tavor_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibsrq->device); - struct mthca_srq *srq = to_msrq(ibsrq); - unsigned long flags; - int err = 0; - int first_ind; - int ind; - int next_ind; - int nreq; - int i; - void *wqe; - void *prev_wqe; - - spin_lock_irqsave(&srq->lock, flags); - - first_ind = srq->first_free; - - for (nreq = 0; wr; wr = wr->next) { - ind = srq->first_free; - wqe = get_wqe(srq, ind); - next_ind = *wqe_to_link(wqe); - - if (unlikely(next_ind < 0)) { - mthca_err(dev, "SRQ %06x full\n", srq->srqn); - err = -ENOMEM; - *bad_wr = wr; - break; - } - - prev_wqe = srq->last; - srq->last = wqe; - - ((struct mthca_next_seg *) wqe)->ee_nds = 0; - /* flags field will always remain 0 */ - - wqe += sizeof (struct mthca_next_seg); - - if (unlikely(wr->num_sge > srq->max_gs)) { - err = -EINVAL; - *bad_wr = wr; - srq->last = prev_wqe; - break; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - } - - if (i < srq->max_gs) - mthca_set_data_seg_inval(wqe); - - ((struct mthca_next_seg *) prev_wqe)->ee_nds = - cpu_to_be32(MTHCA_NEXT_DBD); - - srq->wrid[ind] = wr->wr_id; - srq->first_free = next_ind; - - ++nreq; - if (unlikely(nreq == MTHCA_TAVOR_MAX_WQES_PER_RECV_DB)) { - nreq = 0; - - /* - * Make sure that descriptors are written - * before doorbell is rung. - */ - wmb(); - - mthca_write64(first_ind << srq->wqe_shift, srq->srqn << 8, - dev->kar + MTHCA_RECEIVE_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - - first_ind = srq->first_free; - } - } - - if (likely(nreq)) { - /* - * Make sure that descriptors are written before - * doorbell is rung. - */ - wmb(); - - mthca_write64(first_ind << srq->wqe_shift, (srq->srqn << 8) | nreq, - dev->kar + MTHCA_RECEIVE_DOORBELL, - MTHCA_GET_DOORBELL_LOCK(&dev->doorbell_lock)); - } - - /* - * Make sure doorbells don't leak out of SRQ spinlock and - * reach the HCA out of order: - */ - mmiowb(); - - spin_unlock_irqrestore(&srq->lock, flags); - return err; -} - -int mthca_arbel_post_srq_recv(struct ib_srq *ibsrq, struct ib_recv_wr *wr, - struct ib_recv_wr **bad_wr) -{ - struct mthca_dev *dev = to_mdev(ibsrq->device); - struct mthca_srq *srq = to_msrq(ibsrq); - unsigned long flags; - int err = 0; - int ind; - int next_ind; - int nreq; - int i; - void *wqe; - - spin_lock_irqsave(&srq->lock, flags); - - for (nreq = 0; wr; ++nreq, wr = wr->next) { - ind = srq->first_free; - wqe = get_wqe(srq, ind); - next_ind = *wqe_to_link(wqe); - - if (unlikely(next_ind < 0)) { - mthca_err(dev, "SRQ %06x full\n", srq->srqn); - err = -ENOMEM; - *bad_wr = wr; - break; - } - - ((struct mthca_next_seg *) wqe)->ee_nds = 0; - /* flags field will always remain 0 */ - - wqe += sizeof (struct mthca_next_seg); - - if (unlikely(wr->num_sge > srq->max_gs)) { - err = -EINVAL; - *bad_wr = wr; - break; - } - - for (i = 0; i < wr->num_sge; ++i) { - mthca_set_data_seg(wqe, wr->sg_list + i); - wqe += sizeof (struct mthca_data_seg); - } - - if (i < srq->max_gs) - mthca_set_data_seg_inval(wqe); - - srq->wrid[ind] = wr->wr_id; - srq->first_free = next_ind; - } - - if (likely(nreq)) { - srq->counter += nreq; - - /* - * Make sure that descriptors are written before - * we write doorbell record. - */ - wmb(); - *srq->db = cpu_to_be32(srq->counter); - } - - spin_unlock_irqrestore(&srq->lock, flags); - return err; -} - -int mthca_max_srq_sge(struct mthca_dev *dev) -{ - if (mthca_is_memfree(dev)) - return dev->limits.max_sg; - - /* - * SRQ allocations are based on powers of 2 for Tavor, - * (although they only need to be multiples of 16 bytes). - * - * Therefore, we need to base the max number of sg entries on - * the largest power of 2 descriptor size that is <= to the - * actual max WQE descriptor size, rather than return the - * max_sg value given by the firmware (which is based on WQE - * sizes as multiples of 16, not powers of 2). - * - * If SRQ implementation is changed for Tavor to be based on - * multiples of 16, the calculation below can be deleted and - * the FW max_sg value returned. - */ - return min_t(int, dev->limits.max_sg, - ((1 << (fls(dev->limits.max_desc_sz) - 1)) - - sizeof (struct mthca_next_seg)) / - sizeof (struct mthca_data_seg)); -} - -int mthca_init_srq_table(struct mthca_dev *dev) -{ - int err; - - if (!(dev->mthca_flags & MTHCA_FLAG_SRQ)) - return 0; - - spin_lock_init(&dev->srq_table.lock); - - err = mthca_alloc_init(&dev->srq_table.alloc, - dev->limits.num_srqs, - dev->limits.num_srqs - 1, - dev->limits.reserved_srqs); - if (err) - return err; - - err = mthca_array_init(&dev->srq_table.srq, - dev->limits.num_srqs); - if (err) - mthca_alloc_cleanup(&dev->srq_table.alloc); - - return err; -} - -void mthca_cleanup_srq_table(struct mthca_dev *dev) -{ - if (!(dev->mthca_flags & MTHCA_FLAG_SRQ)) - return; - - mthca_array_cleanup(&dev->srq_table.srq, dev->limits.num_srqs); - mthca_alloc_cleanup(&dev->srq_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_uar.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_uar.c deleted file mode 100644 index 5c9ad23de8d9..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_uar.c +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include - -#include "mthca_dev.h" -#include "mthca_memfree.h" - -int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar) -{ - uar->index = mthca_alloc(&dev->uar_table.alloc); - if (uar->index == -1) - return -ENOMEM; - - uar->pfn = (pci_resource_start(dev->pdev, 2) >> PAGE_SHIFT) + uar->index; - - return 0; -} - -void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar) -{ - mthca_free(&dev->uar_table.alloc, uar->index); -} - -int mthca_init_uar_table(struct mthca_dev *dev) -{ - int ret; - - ret = mthca_alloc_init(&dev->uar_table.alloc, - dev->limits.num_uars, - dev->limits.num_uars - 1, - dev->limits.reserved_uars + 1); - if (ret) - return ret; - - ret = mthca_init_db_tab(dev); - if (ret) - mthca_alloc_cleanup(&dev->uar_table.alloc); - - return ret; -} - -void mthca_cleanup_uar_table(struct mthca_dev *dev) -{ - mthca_cleanup_db_tab(dev); - - /* XXX check if any UARs are still allocated? */ - mthca_alloc_cleanup(&dev->uar_table.alloc); -} diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_user.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_user.h deleted file mode 100644 index 5fe56e810739..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_user.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (c) 2005 Topspin Communications. All rights reserved. - * Copyright (c) 2005, 2006 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_USER_H -#define MTHCA_USER_H - -#include - -/* - * Increment this value if any changes that break userspace ABI - * compatibility are made. - */ -#define MTHCA_UVERBS_ABI_VERSION 1 - -/* - * Make sure that all structs defined in this file remain laid out so - * that they pack the same way on 32-bit and 64-bit architectures (to - * avoid incompatibility between 32-bit userspace and 64-bit kernels). - * In particular do not use pointer types -- pass pointers in __u64 - * instead. - */ - -struct mthca_alloc_ucontext_resp { - __u32 qp_tab_size; - __u32 uarc_size; -}; - -struct mthca_alloc_pd_resp { - __u32 pdn; - __u32 reserved; -}; - -struct mthca_reg_mr { -/* - * Mark the memory region with a DMA attribute that causes - * in-flight DMA to be flushed when the region is written to: - */ -#define MTHCA_MR_DMASYNC 0x1 - __u32 mr_attrs; - __u32 reserved; -}; - -struct mthca_create_cq { - __u32 lkey; - __u32 pdn; - __u64 arm_db_page; - __u64 set_db_page; - __u32 arm_db_index; - __u32 set_db_index; -}; - -struct mthca_create_cq_resp { - __u32 cqn; - __u32 reserved; -}; - -struct mthca_resize_cq { - __u32 lkey; - __u32 reserved; -}; - -struct mthca_create_srq { - __u32 lkey; - __u32 db_index; - __u64 db_page; -}; - -struct mthca_create_srq_resp { - __u32 srqn; - __u32 reserved; -}; - -struct mthca_create_qp { - __u32 lkey; - __u32 reserved; - __u64 sq_db_page; - __u64 rq_db_page; - __u32 sq_db_index; - __u32 rq_db_index; -}; - -#endif /* MTHCA_USER_H */ diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_wqe.h b/sys/ofed/drivers/infiniband/hw/mthca/mthca_wqe.h deleted file mode 100644 index 341a5ae881c1..000000000000 --- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_wqe.h +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) 2005 Cisco Systems. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef MTHCA_WQE_H -#define MTHCA_WQE_H - -#include - -enum { - MTHCA_NEXT_DBD = 1 << 7, - MTHCA_NEXT_FENCE = 1 << 6, - MTHCA_NEXT_CQ_UPDATE = 1 << 3, - MTHCA_NEXT_EVENT_GEN = 1 << 2, - MTHCA_NEXT_SOLICIT = 1 << 1, - MTHCA_NEXT_IP_CSUM = 1 << 4, - MTHCA_NEXT_TCP_UDP_CSUM = 1 << 5, - - MTHCA_MLX_VL15 = 1 << 17, - MTHCA_MLX_SLR = 1 << 16 -}; - -enum { - MTHCA_INVAL_LKEY = 0x100, - MTHCA_TAVOR_MAX_WQES_PER_RECV_DB = 256, - MTHCA_ARBEL_MAX_WQES_PER_SEND_DB = 255 -}; - -struct mthca_next_seg { - __be32 nda_op; /* [31:6] next WQE [4:0] next opcode */ - __be32 ee_nds; /* [31:8] next EE [7] DBD [6] F [5:0] next WQE size */ - __be32 flags; /* [3] CQ [2] Event [1] Solicit */ - __be32 imm; /* immediate data */ -}; - -struct mthca_tavor_ud_seg { - u32 reserved1; - __be32 lkey; - __be64 av_addr; - u32 reserved2[4]; - __be32 dqpn; - __be32 qkey; - u32 reserved3[2]; -}; - -struct mthca_arbel_ud_seg { - __be32 av[8]; - __be32 dqpn; - __be32 qkey; - u32 reserved[2]; -}; - -struct mthca_bind_seg { - __be32 flags; /* [31] Atomic [30] rem write [29] rem read */ - u32 reserved; - __be32 new_rkey; - __be32 lkey; - __be64 addr; - __be64 length; -}; - -struct mthca_raddr_seg { - __be64 raddr; - __be32 rkey; - u32 reserved; -}; - -struct mthca_atomic_seg { - __be64 swap_add; - __be64 compare; -}; - -struct mthca_data_seg { - __be32 byte_count; - __be32 lkey; - __be64 addr; -}; - -struct mthca_mlx_seg { - __be32 nda_op; - __be32 nds; - __be32 flags; /* [17] VL15 [16] SLR [14:12] static rate - [11:8] SL [3] C [2] E */ - __be16 rlid; - __be16 vcrc; -}; - -static __always_inline void mthca_set_data_seg(struct mthca_data_seg *dseg, - struct ib_sge *sg) -{ - dseg->byte_count = cpu_to_be32(sg->length); - dseg->lkey = cpu_to_be32(sg->lkey); - dseg->addr = cpu_to_be64(sg->addr); -} - -static __always_inline void mthca_set_data_seg_inval(struct mthca_data_seg *dseg) -{ - dseg->byte_count = 0; - dseg->lkey = cpu_to_be32(MTHCA_INVAL_LKEY); - dseg->addr = 0; -} - -#endif /* MTHCA_WQE_H */ diff --git a/sys/ofed/drivers/infiniband/util/Kconfig b/sys/ofed/drivers/infiniband/util/Kconfig deleted file mode 100644 index 5e98eaa9c6a6..000000000000 --- a/sys/ofed/drivers/infiniband/util/Kconfig +++ /dev/null @@ -1,6 +0,0 @@ -config INFINIBAND_MADEYE - tristate "MAD debug viewer for InfiniBand" - depends on INFINIBAND - ---help--- - Prints sent and received MADs on QP 0/1 for debugging. - diff --git a/targets/pseudo/userland/lib/Makefile.depend b/targets/pseudo/userland/lib/Makefile.depend index c9a880aec3aa..87d53d0ebc88 100644 --- a/targets/pseudo/userland/lib/Makefile.depend +++ b/targets/pseudo/userland/lib/Makefile.depend @@ -251,7 +251,6 @@ DIRDEPS+= \ contrib/ofed/usr.lib/libibumad \ contrib/ofed/usr.lib/libibverbs \ contrib/ofed/usr.lib/libmlx4 \ - contrib/ofed/usr.lib/libmthca \ contrib/ofed/usr.lib/libopensm \ contrib/ofed/usr.lib/libosmcomp \ contrib/ofed/usr.lib/libosmvendor \