d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add support for fast unload in shutdown flow in mlx5core.

Browse Source 
This patch accumulates the following Linux commits:

- 8812c24d28f4972c4f2b9998bf30b1f2a1b62adf
  net/mlx5: Add fast unload support in shutdown flow
- 59211bd3b6329c3e5f4a90ac3d7f87ffa7867073
  net/mlx5: Split the load/unload flow into hardware and software flows
- 4525abeaae54560254a1bb8970b3d4c225d32ef4
  net/mlx5: Expose command polling interface

Submitted by:	Matthew Finlay <matt@mellanox.com>
MFC after:	1 week
Sponsored by:	Mellanox Technologies
This commit is contained in:
hselasky 2018-03-23 18:02:20 +00:00
parent 947b9c0958
commit 6b367f2d91
7 changed files with 184 additions and 69 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sys/dev/mlx5/driver.h
View File

@ -779,6 +779,7 @@ struct mlx5_cmd_work_ent {
u64 ts2;
u16 op;
u8 busy;
bool polling;
};
struct mlx5_pas {
@ -866,6 +867,7 @@ static inline u32 mlx5_base_mkey(const u32 key)
return key & 0xffffff00u;
}
void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force);
int mlx5_cmd_init(struct mlx5_core_dev *dev);
void mlx5_cmd_cleanup(struct mlx5_core_dev *dev);
void mlx5_cmd_use_events(struct mlx5_core_dev *dev);
@ -877,6 +879,8 @@ int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size, mlx5_cmd_cbk_t callback,
void *context);
int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size);
int mlx5_cmd_alloc_uar(struct mlx5_core_dev *dev, u32 *uarn);
int mlx5_cmd_free_uar(struct mlx5_core_dev *dev, u32 uarn);
int mlx5_alloc_uuars(struct mlx5_core_dev *dev, struct mlx5_uuar_info *uuari);

29
sys/dev/mlx5/mlx5_core/mlx5_cmd.c
View File

@ -859,6 +859,7 @@ static void cmd_work_handler(struct work_struct *work)
unsigned long cb_timeout = msecs_to_jiffies(MLX5_CMD_TIMEOUT_MSEC);
struct mlx5_cmd_layout *lay;
struct semaphore *sem;
bool poll_cmd = ent->polling;
sem = ent->page_queue ? &cmd->pages_sem : &cmd->sem;
down(sem);
@ -897,7 +898,7 @@ static void cmd_work_handler(struct work_struct *work)
iowrite32be(1 << ent->idx, &dev->iseg->cmd_dbell);
mmiowb();
/* if not in polling don't use ent after this point*/
if (cmd->mode == CMD_MODE_POLLING) {
if (cmd->mode == CMD_MODE_POLLING || poll_cmd) {
poll_timeout(ent);
/* make sure we read the descriptor after ownership is SW */
mlx5_cmd_comp_handler(dev, 1U << ent->idx);
@ -940,7 +941,7 @@ static int wait_func(struct mlx5_core_dev *dev, struct mlx5_cmd_work_ent *ent)
struct mlx5_cmd *cmd = &dev->cmd;
int err;
if (cmd->mode == CMD_MODE_POLLING) {
if (cmd->mode == CMD_MODE_POLLING || ent->polling) {
wait_for_completion(&ent->done);
err = ent->ret;
} else if (!wait_for_completion_timeout(&ent->done, timeout)) {
@ -969,7 +970,8 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
int uin_size,
struct mlx5_cmd_msg *out, void *uout, int uout_size,
mlx5_cmd_cbk_t callback,
void *context, int page_queue, u8 *status)
void *context, int page_queue, u8 *status,
bool force_polling)
{
struct mlx5_cmd *cmd = &dev->cmd;
struct mlx5_cmd_work_ent *ent;
@ -986,6 +988,8 @@ static int mlx5_cmd_invoke(struct mlx5_core_dev *dev, struct mlx5_cmd_msg *in,
if (IS_ERR(ent))
return PTR_ERR(ent);
ent->polling = force_polling;
if (!callback)
init_completion(&ent->done);
@ -1260,7 +1264,8 @@ static int is_manage_pages(void *in)
static int cmd_exec_helper(struct mlx5_core_dev *dev,
void *in, int in_size,
void *out, int out_size,
mlx5_cmd_cbk_t callback, void *context)
mlx5_cmd_cbk_t callback, void *context,
bool force_polling)
{
struct mlx5_cmd_msg *inb;
struct mlx5_cmd_msg *outb;
@ -1300,7 +1305,7 @@ static int cmd_exec_helper(struct mlx5_core_dev *dev,
}
err = mlx5_cmd_invoke(dev, inb, in_size, outb, out, out_size, callback,
context, pages_queue, &status);
context, pages_queue, &status, force_polling);
if (err) {
if (err == -ETIMEDOUT)
return err;
@ -1331,7 +1336,7 @@ int mlx5_cmd_exec(struct mlx5_core_dev *dev, void *in, int in_size, void *out,
{
int err;
err = cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL);
err = cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL, false);
return err ? : mlx5_cmd_check(dev, in, out);
}
EXPORT_SYMBOL(mlx5_cmd_exec);
@ -1340,10 +1345,20 @@ int mlx5_cmd_exec_cb(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size, mlx5_cmd_cbk_t callback,
void *context)
{
return cmd_exec_helper(dev, in, in_size, out, out_size, callback, context);
return cmd_exec_helper(dev, in, in_size, out, out_size, callback, context, false);
}
EXPORT_SYMBOL(mlx5_cmd_exec_cb);
int mlx5_cmd_exec_polling(struct mlx5_core_dev *dev, void *in, int in_size,
void *out, int out_size)
{
int err;
err = cmd_exec_helper(dev, in, in_size, out, out_size, NULL, NULL, true);
return err ? : mlx5_cmd_check(dev, in, out);
}
EXPORT_SYMBOL(mlx5_cmd_exec_polling);
static void destroy_msg_cache(struct mlx5_core_dev *dev)
{
struct mlx5_cmd *cmd = &dev->cmd;

3
sys/dev/mlx5/mlx5_core/mlx5_core.h
View File

@ -70,9 +70,10 @@ int mlx5_query_hca_caps(struct mlx5_core_dev *dev);
int mlx5_query_board_id(struct mlx5_core_dev *dev);
int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev);
void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
unsigned long param);
void mlx5_enter_error_state(struct mlx5_core_dev *dev);
void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force);
void mlx5_disable_device(struct mlx5_core_dev *dev);
void mlx5_recover_device(struct mlx5_core_dev *dev);

28
sys/dev/mlx5/mlx5_core/mlx5_fw.c
View File

@ -218,6 +218,34 @@ int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev)
return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}
int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev)
{
u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {0};
u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {0};
int force_state;
int ret;
if (!MLX5_CAP_GEN(dev, force_teardown)) {
mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n");
return -EOPNOTSUPP;
}
MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
MLX5_SET(teardown_hca_in, in, profile, MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE);
ret = mlx5_cmd_exec_polling(dev, in, sizeof(in), out, sizeof(out));
if (ret)
return ret;
force_state = MLX5_GET(teardown_hca_out, out, force_state);
if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) {
mlx5_core_err(dev, "teardown with force mode failed\n");
return -EIO;
}
return 0;
}
int mlx5_core_set_dc_cnak_trace(struct mlx5_core_dev *dev, int enable,
u64 addr)
{

4
sys/dev/mlx5/mlx5_core/mlx5_health.c
View File

@ -90,7 +90,7 @@ static int in_fatal(struct mlx5_core_dev *dev)
return 0;
}
void mlx5_enter_error_state(struct mlx5_core_dev *dev)
void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force)
{
mutex_lock(&dev->intf_state_mutex);
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
@ -99,7 +99,7 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev)
}
mlx5_core_err(dev, "start\n");
if (pci_channel_offline(dev->pdev) || in_fatal(dev)) {
if (pci_channel_offline(dev->pdev) || in_fatal(dev) || force) {
dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
mlx5_trigger_cmd_completions(dev);
}

171
sys/dev/mlx5/mlx5_core/mlx5_main.c
View File

@ -868,7 +868,61 @@ static void mlx5_pci_close(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mlx5_pci_disable_device(dev);
}
static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
{
struct pci_dev *pdev = dev->pdev;
int err;
err = mlx5_query_hca_caps(dev);
if (err) {
dev_err(&pdev->dev, "query hca failed\n");
goto out;
}
err = mlx5_query_board_id(dev);
if (err) {
dev_err(&pdev->dev, "query board id failed\n");
goto out;
}
err = mlx5_eq_init(dev);
if (err) {
dev_err(&pdev->dev, "failed to initialize eq\n");
goto out;
}
MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
err = mlx5_init_cq_table(dev);
if (err) {
dev_err(&pdev->dev, "failed to initialize cq table\n");
goto err_eq_cleanup;
}
mlx5_init_qp_table(dev);
mlx5_init_srq_table(dev);
mlx5_init_mr_table(dev);
return 0;
err_eq_cleanup:
mlx5_eq_cleanup(dev);
out:
return err;
}
static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
{
mlx5_cleanup_mr_table(dev);
mlx5_cleanup_srq_table(dev);
mlx5_cleanup_qp_table(dev);
mlx5_cleanup_cq_table(dev);
mlx5_eq_cleanup(dev);
}
static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
bool boot)
{
struct pci_dev *pdev = dev->pdev;
int err;
@ -900,12 +954,10 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
goto err_cmd_cleanup;
}
mlx5_pagealloc_init(dev);
err = mlx5_core_enable_hca(dev);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""enable hca failed\n");
goto err_pagealloc_cleanup;
goto err_cmd_cleanup;
}
err = mlx5_core_set_issi(dev);
@ -958,34 +1010,21 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mlx5_start_health_poll(dev);
err = mlx5_query_hca_caps(dev);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""query hca failed\n");
goto err_stop_poll;
}
err = mlx5_query_board_id(dev);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""query board id failed\n");
if (boot && mlx5_init_once(dev, priv)) {
dev_err(&pdev->dev, "sw objs init failed\n");
goto err_stop_poll;
}
err = mlx5_enable_msix(dev);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""enable msix failed\n");
goto err_stop_poll;
}
err = mlx5_eq_init(dev);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""failed to initialize eq\n");
goto disable_msix;
goto err_cleanup_once;
}
err = mlx5_alloc_uuars(dev, &priv->uuari);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""Failed allocating uar, aborting\n");
goto err_eq_cleanup;
goto err_disable_msix;
}
err = mlx5_start_eqs(dev);
@ -1003,23 +1042,16 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
if (map_bf_area(dev))
device_printf((&pdev->dev)->bsddev, "ERR: ""Failed to map blue flame area\n");
MLX5_INIT_DOORBELL_LOCK(&priv->cq_uar_lock);
mlx5_init_cq_table(dev);
mlx5_init_qp_table(dev);
mlx5_init_srq_table(dev);
mlx5_init_mr_table(dev);
err = mlx5_init_fs(dev);
if (err) {
mlx5_core_err(dev, "flow steering init %d\n", err);
goto err_init_tables;
goto err_free_comp_eqs;
}
err = mlx5_register_device(dev);
if (err) {
dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err);
goto err_reg_dev;
goto err_fs;
}
mlx5_fwdump_prep(dev);
@ -1031,13 +1063,11 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mutex_unlock(&dev->intf_state_mutex);
return 0;
err_reg_dev:
err_fs:
mlx5_cleanup_fs(dev);
err_init_tables:
mlx5_cleanup_mr_table(dev);
mlx5_cleanup_srq_table(dev);
mlx5_cleanup_qp_table(dev);
mlx5_cleanup_cq_table(dev);
err_free_comp_eqs:
free_comp_eqs(dev);
unmap_bf_area(dev);
err_stop_eqs:
@ -1046,12 +1076,13 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
err_free_uar:
mlx5_free_uuars(dev, &priv->uuari);
err_eq_cleanup:
mlx5_eq_cleanup(dev);
disable_msix:
err_disable_msix:
mlx5_disable_msix(dev);
err_cleanup_once:
if (boot)
mlx5_cleanup_once(dev);
err_stop_poll:
mlx5_stop_health_poll(dev);
if (mlx5_cmd_teardown_hca(dev)) {
@ -1068,9 +1099,6 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
err_disable_hca:
mlx5_core_disable_hca(dev);
err_pagealloc_cleanup:
mlx5_pagealloc_cleanup(dev);
err_cmd_cleanup:
mlx5_cmd_cleanup(dev);
@ -1081,13 +1109,16 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
return err;
}
static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv,
bool cleanup)
{
int err = 0;
mutex_lock(&dev->intf_state_mutex);
if (test_bit(MLX5_INTERFACE_STATE_DOWN, &dev->intf_state)) {
dev_warn(&dev->pdev->dev, "%s: interface is down, NOP\n", __func__);
if (cleanup)
mlx5_cleanup_once(dev);
goto out;
}
@ -1095,17 +1126,14 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mlx5_unregister_device(dev);
mlx5_cleanup_fs(dev);
mlx5_cleanup_mr_table(dev);
mlx5_cleanup_srq_table(dev);
mlx5_cleanup_qp_table(dev);
mlx5_cleanup_cq_table(dev);
unmap_bf_area(dev);
mlx5_wait_for_reclaim_vfs_pages(dev);
free_comp_eqs(dev);
mlx5_stop_eqs(dev);
mlx5_free_uuars(dev, &priv->uuari);
mlx5_eq_cleanup(dev);
mlx5_disable_msix(dev);
if (cleanup)
mlx5_cleanup_once(dev);
mlx5_stop_health_poll(dev);
err = mlx5_cmd_teardown_hca(dev);
if (err) {
@ -1115,7 +1143,6 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv)
mlx5_pagealloc_stop(dev);
mlx5_reclaim_startup_pages(dev);
mlx5_core_disable_hca(dev);
mlx5_pagealloc_cleanup(dev);
mlx5_cmd_cleanup(dev);
out:
@ -1184,7 +1211,9 @@ static int init_one(struct pci_dev *pdev,
goto close_pci;
}
err = mlx5_load_one(dev, priv);
mlx5_pagealloc_init(dev);
err = mlx5_load_one(dev, priv, true);
if (err) {
device_printf((&pdev->dev)->bsddev, "ERR: ""mlx5_register_device failed %d\n", err);
goto clean_health;
@ -1193,6 +1222,7 @@ static int init_one(struct pci_dev *pdev,
return 0;
clean_health:
mlx5_pagealloc_cleanup(dev);
mlx5_health_cleanup(dev);
close_pci:
mlx5_pci_close(dev, priv);
@ -1206,12 +1236,13 @@ static void remove_one(struct pci_dev *pdev)
struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
struct mlx5_priv *priv = &dev->priv;
if (mlx5_unload_one(dev, priv)) {
if (mlx5_unload_one(dev, priv, true)) {
dev_err(&dev->pdev->dev, "mlx5_unload_one failed\n");
mlx5_health_cleanup(dev);
return;
}
mlx5_pagealloc_cleanup(dev);
mlx5_health_cleanup(dev);
mlx5_pci_close(dev, priv);
pci_set_drvdata(pdev, NULL);
@ -1225,8 +1256,8 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
struct mlx5_priv *priv = &dev->priv;
dev_info(&pdev->dev, "%s was called\n", __func__);
mlx5_enter_error_state(dev);
mlx5_unload_one(dev, priv);
mlx5_enter_error_state(dev, false);
mlx5_unload_one(dev, priv, false);
if (state) {
pci_save_state(pdev->dev.bsddev);
mlx5_drain_health_wq(dev);
@ -1310,7 +1341,7 @@ static void mlx5_pci_resume(struct pci_dev *pdev)
pci_save_state(pdev->dev.bsddev);
wait_vital(pdev);
err = mlx5_load_one(dev, priv);
err = mlx5_load_one(dev, priv, false);
if (err)
dev_err(&pdev->dev, "%s: mlx5_load_one failed with error code: %d\n"
, __func__, err);
@ -1324,13 +1355,41 @@ static const struct pci_error_handlers mlx5_err_handler = {
.resume = mlx5_pci_resume
};
static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
{
int err;
if (!MLX5_CAP_GEN(dev, force_teardown)) {
mlx5_core_dbg(dev, "force teardown is not supported in the firmware\n");
return -EOPNOTSUPP;
}
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
return -EAGAIN;
}
err = mlx5_cmd_force_teardown_hca(dev);
if (err) {
mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err);
return err;
}
mlx5_enter_error_state(dev, true);
return 0;
}
static void shutdown_one(struct pci_dev *pdev)
{
struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
struct mlx5_priv *priv = &dev->priv;
int err;
set_bit(MLX5_INTERFACE_STATE_SHUTDOWN, &dev->intf_state);
mlx5_unload_one(dev, priv);
err = mlx5_try_fast_unload(dev);
if (err)
mlx5_unload_one(dev, priv, false);
mlx5_pci_disable_device(dev);
}

14
sys/dev/mlx5/mlx5_ifc.h
View File

@ -941,7 +941,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 max_indirection[0x8];
u8 reserved_8[0x1];
u8 log_max_mrw_sz[0x7];
u8 reserved_9[0x2];
u8 force_teardown[0x1];
u8 reserved_9[0x1];
u8 log_max_bsf_list_size[0x6];
u8 reserved_10[0x2];
u8 log_max_klm_list_size[0x6];
@ -3149,18 +3150,25 @@ struct mlx5_ifc_icmd_access_reg_in_bits {
u8 register_data[0][0x20];
};
enum {
MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_SUCCESS = 0x0,
MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL = 0x1,
};
struct mlx5_ifc_teardown_hca_out_bits {
u8 status[0x8];
u8 reserved_0[0x18];
u8 syndrome[0x20];
u8 reserved_1[0x40];
u8 reserved_1[0x3f];
u8 force_state[0x1];
};
enum {
MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE = 0x0,
MLX5_TEARDOWN_HCA_IN_PROFILE_PANIC_CLOSE = 0x1,
MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE = 0x1,
};
struct mlx5_ifc_teardown_hca_in_bits {