d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor access to CR-space into using VSC APIs in mlx5core.

Browse Source 
Remove no longer used files and APIs.

MFC after:		1 week
Sponsored by:		Mellanox Technologies
This commit is contained in:
Hans Petter Selasky 2018-07-17 10:16:32 +00:00
parent 9fc929d2e2
commit b575d8c850
8 changed files with 92 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sys/conf/files
View File

@ -4745,8 +4745,6 @@ dev/mlx5/mlx5_core/mlx5_cmd.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_cq.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_crspace.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_diagnostics.c optional mlx5 pci \
compile-with "${OFED_C}"
dev/mlx5/mlx5_core/mlx5_eq.c optional mlx5 pci \

6
sys/dev/mlx5/driver.h
View File

@ -682,7 +682,6 @@ struct mlx5_core_dev {
struct mlx5_flow_root_namespace *sniffer_tx_root_ns;
u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER];
struct mlx5_dump_data *dump_data;
u32 vsec_addr;
};
enum {
@ -1055,8 +1054,11 @@ int mlx5_vsc_find_cap(struct mlx5_core_dev *mdev);
int mlx5_vsc_lock(struct mlx5_core_dev *mdev);
void mlx5_vsc_unlock(struct mlx5_core_dev *mdev);
int mlx5_vsc_set_space(struct mlx5_core_dev *mdev, u16 space);
int mlx5_vsc_write(struct mlx5_core_dev *mdev, u32 addr, u32 *data);
int mlx5_vsc_write(struct mlx5_core_dev *mdev, u32 addr, const u32 *data);
int mlx5_vsc_read(struct mlx5_core_dev *mdev, u32 addr, u32 *data);
int mlx5_vsc_lock_addr_space(struct mlx5_core_dev *mdev, u32 addr);
int mlx5_vsc_unlock_addr_space(struct mlx5_core_dev *mdev, u32 addr);
static inline u32 mlx5_mkey_to_idx(u32 mkey)
{
return mkey >> 8;

10
sys/dev/mlx5/mlx5_core/mlx5_core.h
View File

@ -70,12 +70,6 @@ enum mlx5_semaphore_space_address {
MLX5_SEMAPHORE_SW_RESET = 0x20,
};
enum {
UNLOCK = 0,
LOCK = 1,
CAP_ID = 0x9,
};
struct mlx5_core_dev;
int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
@ -109,8 +103,4 @@ struct mlx5_crspace_regmap {
extern struct pci_driver mlx5_core_driver;
void mlx5_vsec_init(struct mlx5_core_dev *dev);
int mlx5_pciconf_cap9_sem(struct mlx5_core_dev *dev, int state);
int mlx5_pciconf_set_sem_addr_space(struct mlx5_core_dev *dev,
u32 sem_space_address, int state);
#endif /* __MLX5_CORE_H__ */

254
sys/dev/mlx5/mlx5_core/mlx5_crspace.c
View File

@ -1,254 +0,0 @@
/*-
* Copyright (c) 2013-2018, Mellanox Technologies, Ltd. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <linux/pci.h>
#include <linux/delay.h>
#include <dev/mlx5/driver.h>
#include "mlx5_core.h"
enum {
PCI_CTRL_OFFSET = 0x4,
PCI_COUNTER_OFFSET = 0x8,
PCI_SEMAPHORE_OFFSET = 0xc,
PCI_ADDR_OFFSET = 0x10,
PCI_DATA_OFFSET = 0x14,
PCI_FLAG_BIT_OFFS = 31,
PCI_SPACE_BIT_OFFS = 0,
PCI_SPACE_BIT_LEN = 16,
PCI_SIZE_VLD_BIT_OFFS = 28,
PCI_SIZE_VLD_BIT_LEN = 1,
PCI_STATUS_BIT_OFFS = 29,
PCI_STATUS_BIT_LEN = 3,
};
enum {
IFC_MAX_RETRIES = 2048
};
#define MLX5_EXTRACT_C(source, offset, size) \
((((unsigned)(source)) >> (offset)) & MLX5_ONES32(size))
#define MLX5_EXTRACT(src, start, len) \
(((len) == 32) ? (src) : MLX5_EXTRACT_C(src, start, len))
#define MLX5_ONES32(size) \
((size) ? (0xffffffff >> (32 - (size))) : 0)
#define MLX5_MASK32(offset, size) \
(MLX5_ONES32(size) << (offset))
#define MLX5_MERGE_C(rsrc1, rsrc2, start, len) \
((((rsrc2) << (start)) & (MLX5_MASK32((start), (len)))) | \
((rsrc1) & (~MLX5_MASK32((start), (len)))))
#define MLX5_MERGE(rsrc1, rsrc2, start, len) \
(((len) == 32) ? (rsrc2) : MLX5_MERGE_C(rsrc1, rsrc2, start, len))
#define MLX5_SEMAPHORE_SPACE_DOMAIN 0xA
static int mlx5_pciconf_wait_on_flag(struct mlx5_core_dev *dev,
u8 expected_val)
{
int retries = 0;
u32 flag;
for(;;) {
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_ADDR_OFFSET, &flag);
flag = MLX5_EXTRACT(flag, PCI_FLAG_BIT_OFFS, 1);
if (flag == expected_val)
return (0);
retries++;
if (retries > IFC_MAX_RETRIES)
return (-EBUSY);
if ((retries & 0xf) == 0)
usleep_range(1000, 2000);
}
}
static int mlx5_pciconf_read(struct mlx5_core_dev *dev,
unsigned int offset, u32 *data)
{
u32 address;
int ret;
if (MLX5_EXTRACT(offset, 31, 1))
return -EINVAL;
address = MLX5_MERGE(offset, 0, PCI_FLAG_BIT_OFFS, 1);
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_ADDR_OFFSET, address);
ret = mlx5_pciconf_wait_on_flag(dev, 1);
if (ret)
return (ret);
return pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_DATA_OFFSET, data);
}
static int mlx5_pciconf_write(struct mlx5_core_dev *dev,
unsigned int offset, u32 data)
{
u32 address;
if (MLX5_EXTRACT(offset, 31, 1))
return -EINVAL;
/* Set flag to 0x1 */
address = MLX5_MERGE(offset, 1, PCI_FLAG_BIT_OFFS, 1);
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_DATA_OFFSET, data);
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_ADDR_OFFSET, address);
/* Wait for the flag to be cleared */
return mlx5_pciconf_wait_on_flag(dev, 0);
}
int mlx5_pciconf_cap9_sem(struct mlx5_core_dev *dev, int state)
{
u32 counter = 0;
int retries = 0;
u32 lock_val;
if (!dev->vsec_addr)
return -ENXIO;
if (state == UNLOCK) {
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_SEMAPHORE_OFFSET, 0);
return (0);
}
do {
if (retries > IFC_MAX_RETRIES * 10)
return -EBUSY;
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_SEMAPHORE_OFFSET, &lock_val);
if (lock_val != 0) {
retries++;
if (retries > IFC_MAX_RETRIES * 10)
return -EBUSY;
usleep_range(1000, 2000);
continue;
}
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_COUNTER_OFFSET, &counter);
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_SEMAPHORE_OFFSET, counter);
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_SEMAPHORE_OFFSET, &lock_val);
retries++;
} while (counter != lock_val);
return 0;
}
static int mlx5_pciconf_set_addr_space(struct mlx5_core_dev *dev,
u16 space)
{
u32 val;
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_CTRL_OFFSET, &val);
val = MLX5_MERGE(val, space, PCI_SPACE_BIT_OFFS,
PCI_SPACE_BIT_LEN);
pci_write_config_dword(dev->pdev, dev->vsec_addr +
PCI_CTRL_OFFSET, val);
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_CTRL_OFFSET, &val);
if (MLX5_EXTRACT(val, PCI_STATUS_BIT_OFFS,
PCI_STATUS_BIT_LEN) == 0)
return -EINVAL;
return 0;
}
static int mlx5_get_vendor_cap_addr(struct mlx5_core_dev *dev)
{
int vend_cap;
int ret;
vend_cap = pci_find_capability(dev->pdev, CAP_ID);
if (!vend_cap)
return 0;
dev->vsec_addr = vend_cap;
ret = mlx5_pciconf_cap9_sem(dev, LOCK);
if (ret) {
mlx5_core_warn(dev,
"pciconf_cap9_sem locking failure\n");
return 0;
}
if (mlx5_pciconf_set_addr_space(dev,
MLX5_SEMAPHORE_SPACE_DOMAIN))
vend_cap = 0;
ret = mlx5_pciconf_cap9_sem(dev, UNLOCK);
if (ret)
mlx5_core_warn(dev,
"pciconf_cap9_sem unlocking failure\n");
return vend_cap;
}
int mlx5_pciconf_set_sem_addr_space(struct mlx5_core_dev *dev,
u32 sem_space_address, int state)
{
u32 data, id = 0;
int ret;
if (!dev->vsec_addr)
return -ENXIO;
ret = mlx5_pciconf_set_addr_space(dev,
MLX5_SEMAPHORE_SPACE_DOMAIN);
if (ret)
return (ret);
if (state == LOCK)
/* Get a unique ID based on the counter */
pci_read_config_dword(dev->pdev, dev->vsec_addr +
PCI_COUNTER_OFFSET, &id);
/* Try to modify lock */
ret = mlx5_pciconf_write(dev, sem_space_address, id);
if (ret)
return (ret);
/* Verify lock was modified */
ret = mlx5_pciconf_read(dev, sem_space_address, &data);
if (ret)
return -EINVAL;
if (data != id)
return -EBUSY;
return 0;
}
void mlx5_vsec_init(struct mlx5_core_dev *dev)
{
dev->vsec_addr = mlx5_get_vendor_cap_addr(dev);
}

39
sys/dev/mlx5/mlx5_core/mlx5_health.c
View File

@ -59,19 +59,19 @@ enum {
MLX5_SENSOR_FW_SYND_RFR = 5,
};
static int lock_sem_sw_reset(struct mlx5_core_dev *dev, int state)
static int lock_sem_sw_reset(struct mlx5_core_dev *dev)
{
int ret, err;
int ret;
/* Lock GW access */
ret = mlx5_pciconf_cap9_sem(dev, LOCK);
ret = -mlx5_vsc_lock(dev);
if (ret) {
mlx5_core_warn(dev, "Timed out locking gateway %d, %d\n", state, ret);
mlx5_core_warn(dev, "Timed out locking gateway %d\n", ret);
return ret;
}
ret = mlx5_pciconf_set_sem_addr_space(dev, MLX5_SEMAPHORE_SW_RESET, state);
if (ret && state == LOCK) {
ret = -mlx5_vsc_lock_addr_space(dev, MLX5_SEMAPHORE_SW_RESET);
if (ret) {
if (ret == -EBUSY)
mlx5_core_dbg(dev, "SW reset FW semaphore already locked, another function will handle the reset\n");
else
@ -79,9 +79,26 @@ static int lock_sem_sw_reset(struct mlx5_core_dev *dev, int state)
}
/* Unlock GW access */
err = mlx5_pciconf_cap9_sem(dev, UNLOCK);
if (err)
mlx5_core_warn(dev, "Timed out unlocking gateway: state %d, err %d\n", state, err);
mlx5_vsc_unlock(dev);
return ret;
}
static int unlock_sem_sw_reset(struct mlx5_core_dev *dev)
{
int ret;
/* Lock GW access */
ret = -mlx5_vsc_lock(dev);
if (ret) {
mlx5_core_warn(dev, "Timed out locking gateway %d\n", ret);
return ret;
}
ret = -mlx5_vsc_unlock_addr_space(dev, MLX5_SEMAPHORE_SW_RESET);
/* Unlock GW access */
mlx5_vsc_unlock(dev);
return ret;
}
@ -223,7 +240,7 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
if (fatal_error == MLX5_SENSOR_FW_SYND_RFR) {
/* Get cr-dump and reset FW semaphore */
if (mlx5_core_is_pf(dev))
lock = lock_sem_sw_reset(dev, LOCK);
lock = lock_sem_sw_reset(dev);
/* Execute cr-dump and SW reset */
if (lock != -EBUSY) {
@ -249,7 +266,7 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
/* Release FW semaphore if you are the lock owner */
if (!lock)
lock_sem_sw_reset(dev, UNLOCK);
unlock_sem_sw_reset(dev);
mlx5_core_err(dev, "system error event triggered\n");

4
sys/dev/mlx5/mlx5_core/mlx5_main.c
View File

@ -873,7 +873,9 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
struct pci_dev *pdev = dev->pdev;
int err;
mlx5_vsec_init(dev);
err = mlx5_vsc_find_cap(dev);
if (err)
dev_err(&pdev->dev, "Unable to find vendor specific capabilities\n");
err = mlx5_query_hca_caps(dev);
if (err) {

58
sys/dev/mlx5/mlx5_core/mlx5_vsc.c
View File

@ -29,6 +29,8 @@
#include <dev/mlx5/device.h>
#include <dev/mlx5/mlx5_core/mlx5_core.h>
#define MLX5_SEMAPHORE_SPACE_DOMAIN 0xA
struct mlx5_ifc_vsc_space_bits {
u8 status[0x3];
u8 reserved0[0xd];
@ -139,7 +141,7 @@ int mlx5_vsc_set_space(struct mlx5_core_dev *mdev, u16 space)
return 0;
}
int mlx5_vsc_write(struct mlx5_core_dev *mdev, u32 addr, u32 *data)
int mlx5_vsc_write(struct mlx5_core_dev *mdev, u32 addr, const u32 *data)
{
device_t dev = mdev->pdev->dev.bsddev;
int vsc_addr = mdev->vsc_addr;
@ -189,6 +191,60 @@ int mlx5_vsc_read(struct mlx5_core_dev *mdev, u32 addr, u32 *data)
return 0;
}
int mlx5_vsc_lock_addr_space(struct mlx5_core_dev *mdev, u32 addr)
{
device_t dev = mdev->pdev->dev.bsddev;
int vsc_addr = mdev->vsc_addr;
u32 data;
int ret;
u32 id;
ret = mlx5_vsc_set_space(mdev, MLX5_SEMAPHORE_SPACE_DOMAIN);
if (ret)
return ret;
/* Get a unique ID based on the counter */
id = pci_read_config(dev, vsc_addr + MLX5_VSC_COUNTER_OFFSET, 4);
/* Try to modify lock */
ret = mlx5_vsc_write(mdev, addr, &id);
if (ret)
return ret;
/* Verify */
ret = mlx5_vsc_read(mdev, addr, &data);
if (ret)
return ret;
if (data != id)
return EBUSY;
return 0;
}
int mlx5_vsc_unlock_addr_space(struct mlx5_core_dev *mdev, u32 addr)
{
u32 data = 0;
int ret;
ret = mlx5_vsc_set_space(mdev, MLX5_SEMAPHORE_SPACE_DOMAIN);
if (ret)
return ret;
/* Try to modify lock */
ret = mlx5_vsc_write(mdev, addr, &data);
if (ret)
return ret;
/* Verify */
ret = mlx5_vsc_read(mdev, addr, &data);
if (ret)
return ret;
if (data != 0)
return EBUSY;
return 0;
}
int mlx5_vsc_find_cap(struct mlx5_core_dev *mdev)
{
int *capreg = &mdev->vsc_addr;

1
sys/modules/mlx5/Makefile
View File

@ -6,7 +6,6 @@ SRCS= \
mlx5_alloc.c \
mlx5_cmd.c \
mlx5_cq.c \
mlx5_crspace.c \
mlx5_diagnostics.c \
mlx5_eq.c \
mlx5_fs_cmd.c \