diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h index 07d753cea19d..47e19bfb69d8 100644 --- a/sys/dev/mlx5/device.h +++ b/sys/dev/mlx5/device.h @@ -1034,6 +1034,12 @@ enum mlx5_qcam_feature_groups { #define MLX5_CAP_QCAM_FEATURE(mdev, fld) \ MLX5_GET(qcam_reg, (mdev)->caps.qcam, qos_feature_cap_mask.feature_cap.fld) +#define MLX5_CAP_FPGA(mdev, cap) \ + MLX5_GET(fpga_cap, (mdev)->caps.fpga, cap) + +#define MLX5_CAP64_FPGA(mdev, cap) \ + MLX5_GET64(fpga_cap, (mdev)->caps.fpga, cap) + enum { MLX5_CMD_STAT_OK = 0x0, MLX5_CMD_STAT_INT_ERR = 0x1, diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h index ecf92ea35129..b01b78bcfc24 100644 --- a/sys/dev/mlx5/driver.h +++ b/sys/dev/mlx5/driver.h @@ -40,6 +40,7 @@ #include #include #include +#include #include #include @@ -131,6 +132,10 @@ enum { MLX5_REG_DCBX_PARAM = 0x4020, MLX5_REG_DCBX_APP = 0x4021, MLX5_REG_PCAP = 0x5001, + MLX5_REG_FPGA_CAP = 0x4022, + MLX5_REG_FPGA_CTRL = 0x4023, + MLX5_REG_FPGA_ACCESS_REG = 0x4024, + MLX5_REG_FPGA_SHELL_CNTR = 0x4025, MLX5_REG_PMTU = 0x5003, MLX5_REG_PTYS = 0x5004, MLX5_REG_PAOS = 0x5006, @@ -404,6 +409,13 @@ struct mlx5_buf { u8 load_done; }; +struct mlx5_frag_buf { + struct mlx5_buf_list *frags; + int npages; + int size; + u8 page_shift; +}; + struct mlx5_eq { struct mlx5_core_dev *dev; __be32 __iomem *doorbell; @@ -442,6 +454,20 @@ struct mlx5_core_sig_ctx { u32 sigerr_count; }; +enum { + MLX5_MKEY_MR = 1, + MLX5_MKEY_MW, + MLX5_MKEY_MR_USER, +}; + +struct mlx5_core_mkey { + u64 iova; + u64 size; + u32 key; + u32 pd; + u32 type; +}; + struct mlx5_core_mr { u64 iova; u64 size; @@ -645,6 +671,14 @@ enum mlx5_pci_status { MLX5_PCI_STATUS_ENABLED, }; +#define MLX5_MAX_RESERVED_GIDS 8 + +struct mlx5_rsvd_gids { + unsigned int start; + unsigned int count; + struct ida ida; +}; + struct mlx5_special_contexts { int resd_lkey; }; @@ -663,6 +697,7 @@ struct mlx5_core_dev { u32 hca_caps_max[MLX5_CAP_NUM][MLX5_UN_SZ_DW(hca_cap_union)]; struct { u32 qcam[MLX5_ST_SZ_DW(qcam_reg)]; + u32 fpga[MLX5_ST_SZ_DW(fpga_cap)]; } caps; phys_addr_t iseg_base; struct mlx5_init_seg __iomem *iseg; @@ -691,6 +726,14 @@ struct mlx5_core_dev { struct sysctl_ctx_list sysctl_ctx; int msix_eqvec; + + struct { + struct mlx5_rsvd_gids reserved_gids; + atomic_t roce_en; + } roce; +#ifdef CONFIG_MLX5_FPGA + struct mlx5_fpga_device *fpga; +#endif }; enum { @@ -1106,6 +1149,11 @@ void *mlx5_get_protocol_dev(struct mlx5_core_dev *mdev, int protocol); int mlx5_register_interface(struct mlx5_interface *intf); void mlx5_unregister_interface(struct mlx5_interface *intf); +unsigned int mlx5_core_reserved_gids_count(struct mlx5_core_dev *dev); +int mlx5_core_roce_gid_set(struct mlx5_core_dev *dev, unsigned int index, + u8 roce_version, u8 roce_l3_type, const u8 *gid, + const u8 *mac, bool vlan, u16 vlan_id); + struct mlx5_profile { u64 mask; u8 log_max_qp; diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h new file mode 100644 index 000000000000..55008b2bbcf5 --- /dev/null +++ b/sys/dev/mlx5/mlx5_accel/ipsec.h @@ -0,0 +1,139 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_ACCEL_IPSEC_H__ +#define __MLX5_ACCEL_IPSEC_H__ + +#ifdef CONFIG_MLX5_ACCEL + +#include + +enum { + MLX5_ACCEL_IPSEC_DEVICE = BIT(1), + MLX5_ACCEL_IPSEC_IPV6 = BIT(2), + MLX5_ACCEL_IPSEC_ESP = BIT(3), + MLX5_ACCEL_IPSEC_LSO = BIT(4), +}; + +#define MLX5_IPSEC_SADB_IP_AH BIT(7) +#define MLX5_IPSEC_SADB_IP_ESP BIT(6) +#define MLX5_IPSEC_SADB_SA_VALID BIT(5) +#define MLX5_IPSEC_SADB_SPI_EN BIT(4) +#define MLX5_IPSEC_SADB_DIR_SX BIT(3) +#define MLX5_IPSEC_SADB_IPV6 BIT(2) + +enum { + MLX5_IPSEC_CMD_ADD_SA = 0, + MLX5_IPSEC_CMD_DEL_SA = 1, +}; + +enum mlx5_accel_ipsec_enc_mode { + MLX5_IPSEC_SADB_MODE_NONE = 0, + MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1, + MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3, +}; + +#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \ + MLX5_ACCEL_IPSEC_DEVICE) + +struct mlx5_accel_ipsec_sa { + __be32 cmd; + u8 key_enc[32]; + u8 key_auth[32]; + __be32 sip[4]; + __be32 dip[4]; + union { + struct { + __be32 reserved; + u8 salt_iv[8]; + __be32 salt; + } __packed gcm; + struct { + u8 salt[16]; + } __packed cbc; + }; + __be32 spi; + __be32 sw_sa_handle; + __be16 tfclen; + u8 enc_mode; + u8 sip_masklen; + u8 dip_masklen; + u8 flags; + u8 reserved[2]; +} __packed; + +/** + * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command + * @mdev: mlx5 device + * @cmd: command to execute + * May be called from atomic context. Returns context pointer, or error + * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic + * context, to cleanup the context pointer + */ +void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev, + struct mlx5_accel_ipsec_sa *cmd); + +/** + * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion + * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec + * Sleeps (killable) until command execution is complete. + * Returns the command result, or -EINTR if killed + */ +int mlx5_accel_ipsec_sa_cmd_wait(void *context); + +u32 mlx5_accel_ipsec_device_caps(struct mlx5_core_dev *mdev); + +unsigned int mlx5_accel_ipsec_counters_count(struct mlx5_core_dev *mdev); +int mlx5_accel_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, + unsigned int count); + +int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev); +void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev); + +#else + +#define MLX5_IPSEC_DEV(mdev) false + +static inline int mlx5_accel_ipsec_init(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline void mlx5_accel_ipsec_cleanup(struct mlx5_core_dev *mdev) +{ +} + +#endif + +#endif /* __MLX5_ACCEL_IPSEC_H__ */ diff --git a/sys/dev/mlx5/mlx5_core/mlx5_core.h b/sys/dev/mlx5/mlx5_core/mlx5_core.h index 4359a6006968..d24e42dbff00 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_core.h +++ b/sys/dev/mlx5/mlx5_core/mlx5_core.h @@ -85,6 +85,9 @@ void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force); void mlx5_disable_device(struct mlx5_core_dev *dev); void mlx5_recover_device(struct mlx5_core_dev *dev); +int mlx5_register_device(struct mlx5_core_dev *dev); +void mlx5_unregister_device(struct mlx5_core_dev *dev); + void mlx5e_init(void); void mlx5e_cleanup(void); diff --git a/sys/dev/mlx5/mlx5_core/mlx5_main.c b/sys/dev/mlx5/mlx5_core/mlx5_main.c index 302e7c994ccb..8c34b92f9e4e 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_main.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_main.c @@ -41,6 +41,8 @@ #include #include #include +#include +#include #include "mlx5_core.h" #include "fs_core.h" @@ -734,7 +736,8 @@ static void mlx5_remove_device(struct mlx5_interface *intf, struct mlx5_priv *pr } } -static int mlx5_register_device(struct mlx5_core_dev *dev) +int +mlx5_register_device(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; struct mlx5_interface *intf; @@ -748,7 +751,8 @@ static int mlx5_register_device(struct mlx5_core_dev *dev) return 0; } -static void mlx5_unregister_device(struct mlx5_core_dev *dev) +void +mlx5_unregister_device(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; struct mlx5_interface *intf; @@ -912,6 +916,9 @@ static int mlx5_init_once(struct mlx5_core_dev *dev, struct mlx5_priv *priv) mlx5_init_srq_table(dev); mlx5_init_mr_table(dev); + mlx5_init_reserved_gids(dev); + mlx5_fpga_init(dev); + #ifdef RATELIMIT err = mlx5_init_rl_table(dev); if (err) { @@ -941,6 +948,8 @@ static void mlx5_cleanup_once(struct mlx5_core_dev *dev) #ifdef RATELIMIT mlx5_cleanup_rl_table(dev); #endif + mlx5_fpga_cleanup(dev); + mlx5_cleanup_reserved_gids(dev); mlx5_cleanup_mr_table(dev); mlx5_cleanup_srq_table(dev); mlx5_cleanup_qp_table(dev); @@ -1075,6 +1084,12 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, goto err_free_comp_eqs; } + err = mlx5_fpga_device_start(dev); + if (err) { + dev_err(&pdev->dev, "fpga device start failed %d\n", err); + goto err_fpga_start; + } + err = mlx5_register_device(dev); if (err) { dev_err(&pdev->dev, "mlx5_register_device failed %d\n", err); @@ -1088,6 +1103,7 @@ static int mlx5_load_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, mutex_unlock(&dev->intf_state_mutex); return 0; +err_fpga_start: err_fs: mlx5_cleanup_fs(dev); @@ -1152,6 +1168,7 @@ static int mlx5_unload_one(struct mlx5_core_dev *dev, struct mlx5_priv *priv, mlx5_unregister_device(dev); + mlx5_fpga_device_stop(dev); mlx5_cleanup_fs(dev); unmap_bf_area(dev); mlx5_wait_for_reclaim_vfs_pages(dev); diff --git a/sys/dev/mlx5/mlx5_core/wq.h b/sys/dev/mlx5/mlx5_core/wq.h index 01edd5715aa5..13f26b0dd9a1 100644 --- a/sys/dev/mlx5/mlx5_core/wq.h +++ b/sys/dev/mlx5/mlx5_core/wq.h @@ -42,6 +42,12 @@ struct mlx5_wq_ctrl { struct mlx5_db db; }; +struct mlx5_frag_wq_ctrl { + struct mlx5_core_dev *mdev; + struct mlx5_frag_buf frag_buf; + struct mlx5_db db; +}; + struct mlx5_wq_cyc { void *buf; __be32 *db; @@ -49,6 +55,11 @@ struct mlx5_wq_cyc { u8 log_stride; }; +struct mlx5_wq_qp { + struct mlx5_wq_cyc rq; + struct mlx5_wq_cyc sq; +}; + struct mlx5_cqwq { void *buf; __be32 *db; diff --git a/sys/dev/mlx5/mlx5_fpga/cmd.h b/sys/dev/mlx5/mlx5_fpga/cmd.h new file mode 100644 index 000000000000..71a14caf89a9 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/cmd.h @@ -0,0 +1,82 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies, Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_H__ +#define __MLX5_FPGA_H__ + +#include +#include +#include + +enum mlx5_fpga_qpc_field_select { + MLX5_FPGA_QPC_STATE = BIT(0), +}; + +struct mlx5_fpga_qp_counters { + u64 rx_ack_packets; + u64 rx_send_packets; + u64 tx_ack_packets; + u64 tx_send_packets; + u64 rx_total_drop; +}; + +struct mlx5_fpga_shell_counters { + u64 ddr_read_requests; + u64 ddr_write_requests; + u64 ddr_read_bytes; + u64 ddr_write_bytes; +}; + +int mlx5_fpga_caps(struct mlx5_core_dev *dev); +int mlx5_fpga_query(struct mlx5_core_dev *dev, struct mlx5_fpga_query *query); +int mlx5_fpga_ctrl_op(struct mlx5_core_dev *dev, u8 op); +int mlx5_fpga_access_reg(struct mlx5_core_dev *dev, u8 size, u64 addr, + void *buf, bool write); +int mlx5_fpga_sbu_caps(struct mlx5_core_dev *dev, void *caps, int size); +int mlx5_fpga_load(struct mlx5_core_dev *dev, enum mlx5_fpga_image image); +int mlx5_fpga_image_select(struct mlx5_core_dev *dev, + enum mlx5_fpga_image image); +int mlx5_fpga_shell_counters(struct mlx5_core_dev *dev, bool clear, + struct mlx5_fpga_shell_counters *data); + +int mlx5_fpga_create_qp(struct mlx5_core_dev *dev, void *fpga_qpc, + u32 *fpga_qpn); +int mlx5_fpga_modify_qp(struct mlx5_core_dev *dev, u32 fpga_qpn, + enum mlx5_fpga_qpc_field_select fields, void *fpga_qpc); +int mlx5_fpga_query_qp(struct mlx5_core_dev *dev, u32 fpga_qpn, void *fpga_qpc); +int mlx5_fpga_query_qp_counters(struct mlx5_core_dev *dev, u32 fpga_qpn, + bool clear, struct mlx5_fpga_qp_counters *data); +int mlx5_fpga_destroy_qp(struct mlx5_core_dev *dev, u32 fpga_qpn); + +#endif /* __MLX5_FPGA_H__ */ diff --git a/sys/dev/mlx5/mlx5_fpga/conn.h b/sys/dev/mlx5/mlx5_fpga/conn.h new file mode 100644 index 000000000000..d888f1ad311c --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/conn.h @@ -0,0 +1,97 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_CONN_H__ +#define __MLX5_FPGA_CONN_H__ + +#include +#include +#include +#include +#include +#include + +struct mlx5_fpga_conn { + struct mlx5_fpga_device *fdev; + + void (*recv_cb)(void *cb_arg, struct mlx5_fpga_dma_buf *buf); + void *cb_arg; + + /* FPGA QP */ + u32 fpga_qpc[MLX5_ST_SZ_DW(fpga_qpc)]; + u32 fpga_qpn; + + /* CQ */ + struct { + struct mlx5_cqwq wq; + struct mlx5_frag_wq_ctrl wq_ctrl; + struct mlx5_core_cq mcq; + struct tasklet_struct tasklet; + } cq; + + /* QP */ + struct { + bool active; + int sgid_index; + struct mlx5_wq_qp wq; + struct mlx5_wq_ctrl wq_ctrl; + struct mlx5_core_qp mqp; + struct { + spinlock_t lock; /* Protects all SQ state */ + unsigned int pc; + unsigned int cc; + unsigned int size; + struct mlx5_fpga_dma_buf **bufs; + struct list_head backlog; + } sq; + struct { + unsigned int pc; + unsigned int cc; + unsigned int size; + struct mlx5_fpga_dma_buf **bufs; + } rq; + } qp; +}; + +int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev); +void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev); +struct mlx5_fpga_conn * +mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_conn_attr *attr, + enum mlx5_ifc_fpga_qp_type qp_type); +void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn); +int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf); + +#endif /* __MLX5_FPGA_CONN_H__ */ diff --git a/sys/dev/mlx5/mlx5_fpga/core.h b/sys/dev/mlx5/mlx5_fpga/core.h new file mode 100644 index 000000000000..7d591d39c00e --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/core.h @@ -0,0 +1,140 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies, Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_CORE_H__ +#define __MLX5_FPGA_CORE_H__ + +#ifdef CONFIG_MLX5_FPGA + +#include +#include + +/* Represents client-specific and Innova device-specific information */ +struct mlx5_fpga_client_data { + struct list_head list; + struct mlx5_fpga_client *client; + void *data; + bool added; +}; + +enum mlx5_fdev_state { + MLX5_FDEV_STATE_SUCCESS = 0, + MLX5_FDEV_STATE_FAILURE = 1, + MLX5_FDEV_STATE_IN_PROGRESS = 2, + MLX5_FDEV_STATE_NONE = 0xFFFF, +}; + +/* Represents an Innova device */ +struct mlx5_fpga_device { + struct mlx5_core_dev *mdev; + struct completion load_event; + spinlock_t state_lock; /* Protects state transitions */ + enum mlx5_fdev_state fdev_state; + enum mlx5_fpga_status image_status; + enum mlx5_fpga_image last_admin_image; + enum mlx5_fpga_image last_oper_image; + + /* QP Connection resources */ + struct { + u32 pdn; + struct mlx5_core_mkey mkey; + struct mlx5_uars_page *uar; + } conn_res; + + struct mlx5_fpga_ipsec *ipsec; + + struct list_head list; + struct list_head client_data_list; + + /* Shell Transactions state */ + struct mlx5_fpga_conn *shell_conn; + struct mlx5_fpga_trans_device_state *trans; +}; + +#define mlx5_fpga_dbg(__adev, format, ...) \ + dev_dbg(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) + +#define mlx5_fpga_err(__adev, format, ...) \ + dev_err(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) + +#define mlx5_fpga_warn(__adev, format, ...) \ + dev_warn(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d:(pid %d): " format, \ + __func__, __LINE__, current->pid, ##__VA_ARGS__) + +#define mlx5_fpga_warn_ratelimited(__adev, format, ...) \ + dev_warn_ratelimited(&(__adev)->mdev->pdev->dev, "FPGA: %s:%d: " \ + format, __func__, __LINE__, ##__VA_ARGS__) + +#define mlx5_fpga_notice(__adev, format, ...) \ + dev_notice(&(__adev)->mdev->pdev->dev, "FPGA: " format, ##__VA_ARGS__) + +#define mlx5_fpga_info(__adev, format, ...) \ + dev_info(&(__adev)->mdev->pdev->dev, "FPGA: " format, ##__VA_ARGS__) + +int mlx5_fpga_init(struct mlx5_core_dev *mdev); +void mlx5_fpga_cleanup(struct mlx5_core_dev *mdev); +int mlx5_fpga_device_start(struct mlx5_core_dev *mdev); +void mlx5_fpga_device_stop(struct mlx5_core_dev *mdev); +void mlx5_fpga_event(struct mlx5_core_dev *mdev, u8 event, void *data); + +#else + +static inline int mlx5_fpga_init(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline void mlx5_fpga_cleanup(struct mlx5_core_dev *mdev) +{ +} + +static inline int mlx5_fpga_device_start(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline void mlx5_fpga_device_stop(struct mlx5_core_dev *mdev) +{ +} + +static inline void mlx5_fpga_event(struct mlx5_core_dev *mdev, u8 event, + void *data) +{ +} + +#endif + +#endif /* __MLX5_FPGA_CORE_H__ */ diff --git a/sys/dev/mlx5/mlx5_fpga/ipsec.h b/sys/dev/mlx5/mlx5_fpga/ipsec.h new file mode 100644 index 000000000000..856d72ec0d41 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/ipsec.h @@ -0,0 +1,95 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_IPSEC_H__ +#define __MLX5_FPGA_IPSEC_H__ + +#include + +#ifdef CONFIG_MLX5_FPGA + +void *mlx5_fpga_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev, + struct mlx5_accel_ipsec_sa *cmd); +int mlx5_fpga_ipsec_sa_cmd_wait(void *context); + +u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev); +unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev); +int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, + unsigned int counters_count); + +int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev); +void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev); + +#else + +static inline void *mlx5_fpga_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev, + struct mlx5_accel_ipsec_sa *cmd) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline int mlx5_fpga_ipsec_sa_cmd_wait(void *context) +{ + return -EOPNOTSUPP; +} + +static inline u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline unsigned int +mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, + u64 *counters) +{ + return 0; +} + +static inline int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev) +{ + return 0; +} + +static inline void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev) +{ +} + +#endif /* CONFIG_MLX5_FPGA */ + +#endif /* __MLX5_FPGA_SADB_H__ */ diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5_ifc_fpga.h b/sys/dev/mlx5/mlx5_fpga/mlx5_ifc_fpga.h new file mode 100644 index 000000000000..7f630d406bd9 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5_ifc_fpga.h @@ -0,0 +1,500 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies, Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef MLX5_IFC_FPGA_H +#define MLX5_IFC_FPGA_H + +enum { + MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX = 0x2c9, +}; + +enum { + MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_EXAMPLE = 0x1, + MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC = 0x2, + MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_TLS = 0x3, +}; + +enum { + MLX5_FPGA_SHELL_CAPS_QP_TYPE_SHELL_QP = 0x1, + MLX5_FPGA_SHELL_CAPS_QP_TYPE_SANDBOX_QP = 0x2, +}; + +struct mlx5_ifc_fpga_shell_caps_bits { + u8 max_num_qps[0x10]; + u8 reserved_at_10[0x8]; + u8 total_rcv_credits[0x8]; + + u8 reserved_at_20[0xe]; + u8 qp_type[0x2]; + u8 reserved_at_30[0x5]; + u8 rae[0x1]; + u8 rwe[0x1]; + u8 rre[0x1]; + u8 reserved_at_38[0x4]; + u8 dc[0x1]; + u8 ud[0x1]; + u8 uc[0x1]; + u8 rc[0x1]; + + u8 reserved_at_40[0x1a]; + u8 log_ddr_size[0x6]; + + u8 max_fpga_qp_msg_size[0x20]; + + u8 reserved_at_80[0x180]; +}; + +struct mlx5_ifc_fpga_cap_bits { + u8 fpga_id[0x8]; + u8 fpga_device[0x18]; + + u8 register_file_ver[0x20]; + + u8 fpga_ctrl_modify[0x1]; + u8 reserved_at_41[0x5]; + u8 access_reg_query_mode[0x2]; + u8 reserved_at_48[0x6]; + u8 access_reg_modify_mode[0x2]; + u8 reserved_at_50[0x10]; + + u8 reserved_at_60[0x20]; + + u8 image_version[0x20]; + + u8 image_date[0x20]; + + u8 image_time[0x20]; + + u8 shell_version[0x20]; + + u8 reserved_at_100[0x80]; + + struct mlx5_ifc_fpga_shell_caps_bits shell_caps; + + u8 reserved_at_380[0x8]; + u8 ieee_vendor_id[0x18]; + + u8 sandbox_product_version[0x10]; + u8 sandbox_product_id[0x10]; + + u8 sandbox_basic_caps[0x20]; + + u8 reserved_at_3e0[0x10]; + u8 sandbox_extended_caps_len[0x10]; + + u8 sandbox_extended_caps_addr[0x40]; + + u8 fpga_ddr_start_addr[0x40]; + + u8 fpga_cr_space_start_addr[0x40]; + + u8 fpga_ddr_size[0x20]; + + u8 fpga_cr_space_size[0x20]; + + u8 reserved_at_500[0x300]; +}; + +enum { + MLX5_FPGA_CTRL_OPERATION_LOAD = 0x1, + MLX5_FPGA_CTRL_OPERATION_RESET = 0x2, + MLX5_FPGA_CTRL_OPERATION_FLASH_SELECT = 0x3, + MLX5_FPGA_CTRL_OPERATION_SANDBOX_BYPASS_ON = 0x4, + MLX5_FPGA_CTRL_OPERATION_SANDBOX_BYPASS_OFF = 0x5, + MLX5_FPGA_CTRL_OPERATION_RESET_SANDBOX = 0x6, +}; + +struct mlx5_ifc_fpga_ctrl_bits { + u8 reserved_at_0[0x8]; + u8 operation[0x8]; + u8 reserved_at_10[0x8]; + u8 status[0x8]; + + u8 reserved_at_20[0x8]; + u8 flash_select_admin[0x8]; + u8 reserved_at_30[0x8]; + u8 flash_select_oper[0x8]; + + u8 reserved_at_40[0x40]; +}; + +enum { + MLX5_FPGA_ERROR_EVENT_SYNDROME_CORRUPTED_DDR = 0x1, + MLX5_FPGA_ERROR_EVENT_SYNDROME_FLASH_TIMEOUT = 0x2, + MLX5_FPGA_ERROR_EVENT_SYNDROME_INTERNAL_LINK_ERROR = 0x3, + MLX5_FPGA_ERROR_EVENT_SYNDROME_WATCHDOG_FAILURE = 0x4, + MLX5_FPGA_ERROR_EVENT_SYNDROME_I2C_FAILURE = 0x5, + MLX5_FPGA_ERROR_EVENT_SYNDROME_IMAGE_CHANGED = 0x6, + MLX5_FPGA_ERROR_EVENT_SYNDROME_TEMPERATURE_CRITICAL = 0x7, +}; + +struct mlx5_ifc_fpga_error_event_bits { + u8 reserved_at_0[0x40]; + + u8 reserved_at_40[0x18]; + u8 syndrome[0x8]; + + u8 reserved_at_60[0x80]; +}; + +#define MLX5_FPGA_ACCESS_REG_SIZE_MAX 64 + +struct mlx5_ifc_fpga_access_reg_bits { + u8 reserved_at_0[0x20]; + + u8 reserved_at_20[0x10]; + u8 size[0x10]; + + u8 address[0x40]; + + u8 data[0][0x8]; +}; + +enum mlx5_ifc_fpga_qp_state { + MLX5_FPGA_QPC_STATE_INIT = 0x0, + MLX5_FPGA_QPC_STATE_ACTIVE = 0x1, + MLX5_FPGA_QPC_STATE_ERROR = 0x2, +}; + +enum mlx5_ifc_fpga_qp_type { + MLX5_FPGA_QPC_QP_TYPE_SHELL_QP = 0x0, + MLX5_FPGA_QPC_QP_TYPE_SANDBOX_QP = 0x1, +}; + +enum mlx5_ifc_fpga_qp_service_type { + MLX5_FPGA_QPC_ST_RC = 0x0, +}; + +struct mlx5_ifc_fpga_qpc_bits { + u8 state[0x4]; + u8 reserved_at_4[0x1b]; + u8 qp_type[0x1]; + + u8 reserved_at_20[0x4]; + u8 st[0x4]; + u8 reserved_at_28[0x10]; + u8 traffic_class[0x8]; + + u8 ether_type[0x10]; + u8 prio[0x3]; + u8 dei[0x1]; + u8 vid[0xc]; + + u8 reserved_at_60[0x20]; + + u8 reserved_at_80[0x8]; + u8 next_rcv_psn[0x18]; + + u8 reserved_at_a0[0x8]; + u8 next_send_psn[0x18]; + + u8 reserved_at_c0[0x10]; + u8 pkey[0x10]; + + u8 reserved_at_e0[0x8]; + u8 remote_qpn[0x18]; + + u8 reserved_at_100[0x15]; + u8 rnr_retry[0x3]; + u8 reserved_at_118[0x5]; + u8 retry_count[0x3]; + + u8 reserved_at_120[0x20]; + + u8 reserved_at_140[0x10]; + u8 remote_mac_47_32[0x10]; + + u8 remote_mac_31_0[0x20]; + + u8 remote_ip[16][0x8]; + + u8 reserved_at_200[0x40]; + + u8 reserved_at_240[0x10]; + u8 fpga_mac_47_32[0x10]; + + u8 fpga_mac_31_0[0x20]; + + u8 fpga_ip[16][0x8]; +}; + +struct mlx5_ifc_fpga_create_qp_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x40]; + + struct mlx5_ifc_fpga_qpc_bits fpga_qpc; +}; + +struct mlx5_ifc_fpga_create_qp_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x8]; + u8 fpga_qpn[0x18]; + + u8 reserved_at_60[0x20]; + + struct mlx5_ifc_fpga_qpc_bits fpga_qpc; +}; + +struct mlx5_ifc_fpga_modify_qp_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x8]; + u8 fpga_qpn[0x18]; + + u8 field_select[0x20]; + + struct mlx5_ifc_fpga_qpc_bits fpga_qpc; +}; + +struct mlx5_ifc_fpga_modify_qp_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; +}; + +struct mlx5_ifc_fpga_query_qp_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x8]; + u8 fpga_qpn[0x18]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_fpga_query_qp_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; + + struct mlx5_ifc_fpga_qpc_bits fpga_qpc; +}; + +struct mlx5_ifc_fpga_query_qp_counters_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 clear[0x1]; + u8 reserved_at_41[0x7]; + u8 fpga_qpn[0x18]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_fpga_query_qp_counters_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; + + u8 rx_ack_packets[0x40]; + + u8 rx_send_packets[0x40]; + + u8 tx_ack_packets[0x40]; + + u8 tx_send_packets[0x40]; + + u8 rx_total_drop[0x40]; + + u8 reserved_at_1c0[0x1c0]; +}; + +struct mlx5_ifc_fpga_destroy_qp_in_bits { + u8 opcode[0x10]; + u8 reserved_at_10[0x10]; + + u8 reserved_at_20[0x10]; + u8 op_mod[0x10]; + + u8 reserved_at_40[0x8]; + u8 fpga_qpn[0x18]; + + u8 reserved_at_60[0x20]; +}; + +struct mlx5_ifc_fpga_destroy_qp_out_bits { + u8 status[0x8]; + u8 reserved_at_8[0x18]; + + u8 syndrome[0x20]; + + u8 reserved_at_40[0x40]; +}; + +struct mlx5_ifc_ipsec_extended_cap_bits { + u8 encapsulation[0x20]; + + u8 reserved_0[0x15]; + u8 ipv4_fragment[0x1]; + u8 ipv6[0x1]; + u8 esn[0x1]; + u8 lso[0x1]; + u8 transport_and_tunnel_mode[0x1]; + u8 tunnel_mode[0x1]; + u8 transport_mode[0x1]; + u8 ah_esp[0x1]; + u8 esp[0x1]; + u8 ah[0x1]; + u8 ipv4_options[0x1]; + + u8 auth_alg[0x20]; + + u8 enc_alg[0x20]; + + u8 sa_cap[0x20]; + + u8 reserved_1[0x10]; + u8 number_of_ipsec_counters[0x10]; + + u8 ipsec_counters_addr_low[0x20]; + u8 ipsec_counters_addr_high[0x20]; +}; + +struct mlx5_ifc_ipsec_counters_bits { + u8 dec_in_packets[0x40]; + + u8 dec_out_packets[0x40]; + + u8 dec_bypass_packets[0x40]; + + u8 enc_in_packets[0x40]; + + u8 enc_out_packets[0x40]; + + u8 enc_bypass_packets[0x40]; + + u8 drop_dec_packets[0x40]; + + u8 failed_auth_dec_packets[0x40]; + + u8 drop_enc_packets[0x40]; + + u8 success_add_sa[0x40]; + + u8 fail_add_sa[0x40]; + + u8 success_delete_sa[0x40]; + + u8 fail_delete_sa[0x40]; + + u8 dropped_cmd[0x40]; +}; + +struct mlx5_ifc_fpga_shell_counters_bits { + u8 reserved_0[0x20]; + + u8 clear[0x1]; + u8 reserved_1[0x1f]; + + u8 reserved_2[0x40]; + + u8 ddr_read_requests[0x40]; + + u8 ddr_write_requests[0x40]; + + u8 ddr_read_bytes[0x40]; + + u8 ddr_write_bytes[0x40]; + + u8 reserved_3[0x200]; +}; + +enum { + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_READ = 0x0, + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_WRITE = 0x1, + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_READ_RESPONSE = 0x2, + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_WRITE_RESPONSE = 0x3, +}; + +struct mlx5_ifc_fpga_shell_qp_packet_bits { + u8 version[0x4]; + u8 syndrome[0x4]; + u8 reserved_at_8[0x4]; + u8 type[0x4]; + u8 reserved_at_10[0x8]; + u8 tid[0x8]; + + u8 len[0x20]; + + u8 address[0x40]; + + u8 data[0][0x8]; +}; + +enum { + MLX5_FPGA_QP_ERROR_EVENT_SYNDROME_RETRY_COUNTER_EXPIRED = 0x1, + MLX5_FPGA_QP_ERROR_EVENT_SYNDROME_RNR_EXPIRED = 0x2, +}; + +struct mlx5_ifc_fpga_qp_error_event_bits { + u8 reserved_0[0x40]; + + u8 reserved_1[0x18]; + u8 syndrome[0x8]; + + u8 reserved_2[0x60]; + + u8 reserved_3[0x8]; + u8 fpga_qpn[0x18]; +}; + +#endif /* MLX5_IFC_FPGA_H */ diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_cmd.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_cmd.c new file mode 100644 index 000000000000..d71910011a0f --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_cmd.c @@ -0,0 +1,289 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include + +#define MLX5_FPGA_ACCESS_REG_SZ (MLX5_ST_SZ_DW(fpga_access_reg) + \ + MLX5_FPGA_ACCESS_REG_SIZE_MAX) + +int mlx5_fpga_access_reg(struct mlx5_core_dev *dev, u8 size, u64 addr, + void *buf, bool write) +{ + u32 in[MLX5_FPGA_ACCESS_REG_SZ] = {0}; + u32 out[MLX5_FPGA_ACCESS_REG_SZ]; + int err; + + if (size & 3) + return -EINVAL; + if (addr & 3) + return -EINVAL; + if (size > MLX5_FPGA_ACCESS_REG_SIZE_MAX) + return -EINVAL; + + MLX5_SET(fpga_access_reg, in, size, size); + MLX5_SET64(fpga_access_reg, in, address, addr); + if (write) + memcpy(MLX5_ADDR_OF(fpga_access_reg, in, data), buf, size); + + err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_FPGA_ACCESS_REG, 0, write); + if (err) + return err; + + if (!write) + memcpy(buf, MLX5_ADDR_OF(fpga_access_reg, out, data), size); + + return 0; +} + +int mlx5_fpga_caps(struct mlx5_core_dev *dev) +{ + u32 in[MLX5_ST_SZ_DW(fpga_cap)] = {0}; + + return mlx5_core_access_reg(dev, in, sizeof(in), dev->caps.fpga, + MLX5_ST_SZ_BYTES(fpga_cap), + MLX5_REG_FPGA_CAP, 0, 0); +} + +int mlx5_fpga_ctrl_op(struct mlx5_core_dev *dev, u8 op) +{ + u32 in[MLX5_ST_SZ_DW(fpga_ctrl)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_ctrl)]; + + MLX5_SET(fpga_ctrl, in, operation, op); + + return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_FPGA_CTRL, 0, true); +} + +int mlx5_fpga_sbu_caps(struct mlx5_core_dev *dev, void *caps, int size) +{ + unsigned int cap_size = MLX5_CAP_FPGA(dev, sandbox_extended_caps_len); + u64 addr = MLX5_CAP64_FPGA(dev, sandbox_extended_caps_addr); + unsigned int read; + int ret = 0; + + if (cap_size > size) { + mlx5_core_warn(dev, "Not enough buffer %u for FPGA SBU caps %u", + size, cap_size); + return -EINVAL; + } + + while (cap_size > 0) { + read = min_t(unsigned int, cap_size, + MLX5_FPGA_ACCESS_REG_SIZE_MAX); + + ret = mlx5_fpga_access_reg(dev, read, addr, caps, false); + if (ret) { + mlx5_core_warn(dev, "Error reading FPGA SBU caps %u bytes at address %#jx: %d", + read, (uintmax_t)addr, ret); + return ret; + } + + cap_size -= read; + addr += read; + caps += read; + } + + return ret; +} + +static int mlx5_fpga_ctrl_write(struct mlx5_core_dev *dev, u8 op, + enum mlx5_fpga_image image) +{ + u32 in[MLX5_ST_SZ_DW(fpga_ctrl)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_ctrl)]; + + MLX5_SET(fpga_ctrl, in, operation, op); + MLX5_SET(fpga_ctrl, in, flash_select_admin, image); + + return mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_FPGA_CTRL, 0, true); +} + +int mlx5_fpga_load(struct mlx5_core_dev *dev, enum mlx5_fpga_image image) +{ + return mlx5_fpga_ctrl_write(dev, MLX5_FPGA_CTRL_OPERATION_LOAD, image); +} + +int mlx5_fpga_image_select(struct mlx5_core_dev *dev, + enum mlx5_fpga_image image) +{ + return mlx5_fpga_ctrl_write(dev, MLX5_FPGA_CTRL_OPERATION_FLASH_SELECT, image); +} + +int mlx5_fpga_query(struct mlx5_core_dev *dev, struct mlx5_fpga_query *query) +{ + u32 in[MLX5_ST_SZ_DW(fpga_ctrl)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_ctrl)]; + int err; + + err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_FPGA_CTRL, 0, false); + if (err) + return err; + + query->image_status = MLX5_GET(fpga_ctrl, out, status); + query->admin_image = MLX5_GET(fpga_ctrl, out, flash_select_admin); + query->oper_image = MLX5_GET(fpga_ctrl, out, flash_select_oper); + return 0; +} + +int mlx5_fpga_create_qp(struct mlx5_core_dev *dev, void *fpga_qpc, + u32 *fpga_qpn) +{ + u32 in[MLX5_ST_SZ_DW(fpga_create_qp_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_create_qp_out)]; + int ret; + + MLX5_SET(fpga_create_qp_in, in, opcode, MLX5_CMD_OP_FPGA_CREATE_QP); + memcpy(MLX5_ADDR_OF(fpga_create_qp_in, in, fpga_qpc), fpga_qpc, + MLX5_FLD_SZ_BYTES(fpga_create_qp_in, fpga_qpc)); + + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + memcpy(fpga_qpc, MLX5_ADDR_OF(fpga_create_qp_out, out, fpga_qpc), + MLX5_FLD_SZ_BYTES(fpga_create_qp_out, fpga_qpc)); + *fpga_qpn = MLX5_GET(fpga_create_qp_out, out, fpga_qpn); + return ret; +} + +int mlx5_fpga_modify_qp(struct mlx5_core_dev *dev, u32 fpga_qpn, + enum mlx5_fpga_qpc_field_select fields, + void *fpga_qpc) +{ + u32 in[MLX5_ST_SZ_DW(fpga_modify_qp_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_modify_qp_out)]; + + MLX5_SET(fpga_modify_qp_in, in, opcode, MLX5_CMD_OP_FPGA_MODIFY_QP); + MLX5_SET(fpga_modify_qp_in, in, field_select, fields); + MLX5_SET(fpga_modify_qp_in, in, fpga_qpn, fpga_qpn); + memcpy(MLX5_ADDR_OF(fpga_modify_qp_in, in, fpga_qpc), fpga_qpc, + MLX5_FLD_SZ_BYTES(fpga_modify_qp_in, fpga_qpc)); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +int mlx5_fpga_query_qp(struct mlx5_core_dev *dev, + u32 fpga_qpn, void *fpga_qpc) +{ + u32 in[MLX5_ST_SZ_DW(fpga_query_qp_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_query_qp_out)]; + int ret; + + MLX5_SET(fpga_query_qp_in, in, opcode, MLX5_CMD_OP_FPGA_QUERY_QP); + MLX5_SET(fpga_query_qp_in, in, fpga_qpn, fpga_qpn); + + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + memcpy(fpga_qpc, MLX5_ADDR_OF(fpga_query_qp_out, out, fpga_qpc), + MLX5_FLD_SZ_BYTES(fpga_query_qp_out, fpga_qpc)); + return ret; +} + +int mlx5_fpga_destroy_qp(struct mlx5_core_dev *dev, u32 fpga_qpn) +{ + u32 in[MLX5_ST_SZ_DW(fpga_destroy_qp_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_destroy_qp_out)]; + + MLX5_SET(fpga_destroy_qp_in, in, opcode, MLX5_CMD_OP_FPGA_DESTROY_QP); + MLX5_SET(fpga_destroy_qp_in, in, fpga_qpn, fpga_qpn); + + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} + +int mlx5_fpga_query_qp_counters(struct mlx5_core_dev *dev, u32 fpga_qpn, + bool clear, struct mlx5_fpga_qp_counters *data) +{ + u32 in[MLX5_ST_SZ_DW(fpga_query_qp_counters_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_query_qp_counters_out)]; + int ret; + + MLX5_SET(fpga_query_qp_counters_in, in, opcode, + MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS); + MLX5_SET(fpga_query_qp_counters_in, in, clear, clear); + MLX5_SET(fpga_query_qp_counters_in, in, fpga_qpn, fpga_qpn); + + ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (ret) + return ret; + + data->rx_ack_packets = MLX5_GET64(fpga_query_qp_counters_out, out, + rx_ack_packets); + data->rx_send_packets = MLX5_GET64(fpga_query_qp_counters_out, out, + rx_send_packets); + data->tx_ack_packets = MLX5_GET64(fpga_query_qp_counters_out, out, + tx_ack_packets); + data->tx_send_packets = MLX5_GET64(fpga_query_qp_counters_out, out, + tx_send_packets); + data->rx_total_drop = MLX5_GET64(fpga_query_qp_counters_out, out, + rx_total_drop); + + return ret; +} + +int mlx5_fpga_shell_counters(struct mlx5_core_dev *dev, bool clear, + struct mlx5_fpga_shell_counters *data) +{ + u32 in[MLX5_ST_SZ_DW(fpga_shell_counters)] = {0}; + u32 out[MLX5_ST_SZ_DW(fpga_shell_counters)]; + int err; + + MLX5_SET(fpga_shell_counters, in, clear, clear); + err = mlx5_core_access_reg(dev, in, sizeof(in), out, sizeof(out), + MLX5_REG_FPGA_SHELL_CNTR, 0, false); + if (err) + goto out; + if (data) { + data->ddr_read_requests = MLX5_GET64(fpga_shell_counters, out, + ddr_read_requests); + data->ddr_write_requests = MLX5_GET64(fpga_shell_counters, out, + ddr_write_requests); + data->ddr_read_bytes = MLX5_GET64(fpga_shell_counters, out, + ddr_read_bytes); + data->ddr_write_bytes = MLX5_GET64(fpga_shell_counters, out, + ddr_write_bytes); + } + +out: + return err; +} diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_conn.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_conn.c new file mode 100644 index 000000000000..c2a03bbf5717 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_conn.c @@ -0,0 +1,1042 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include + +#define MLX5_FPGA_PKEY 0xFFFF +#define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */ +#define MLX5_FPGA_RECV_SIZE 2048 +#define MLX5_FPGA_PORT_NUM 1 +#define MLX5_FPGA_CQ_BUDGET 64 + +static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ + struct device *dma_device; + int err = 0; + + if (unlikely(!buf->sg[0].data)) + goto out; + + dma_device = &conn->fdev->mdev->pdev->dev; + buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data, + buf->sg[0].size, buf->dma_dir); + err = dma_mapping_error(dma_device, buf->sg[0].dma_addr); + if (unlikely(err)) { + mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err); + err = -ENOMEM; + goto out; + } + + if (!buf->sg[1].data) + goto out; + + buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data, + buf->sg[1].size, buf->dma_dir); + err = dma_mapping_error(dma_device, buf->sg[1].dma_addr); + if (unlikely(err)) { + mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err); + dma_unmap_single(dma_device, buf->sg[0].dma_addr, + buf->sg[0].size, buf->dma_dir); + err = -ENOMEM; + } + +out: + return err; +} + +static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ + struct device *dma_device; + + dma_device = &conn->fdev->mdev->pdev->dev; + if (buf->sg[1].data) + dma_unmap_single(dma_device, buf->sg[1].dma_addr, + buf->sg[1].size, buf->dma_dir); + + if (likely(buf->sg[0].data)) + dma_unmap_single(dma_device, buf->sg[0].dma_addr, + buf->sg[0].size, buf->dma_dir); +} + +static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ + struct mlx5_wqe_data_seg *data; + unsigned int ix; + int err = 0; + + err = mlx5_fpga_conn_map_buf(conn, buf); + if (unlikely(err)) + goto out; + + if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) { + mlx5_fpga_conn_unmap_buf(conn, buf); + return -EBUSY; + } + + ix = conn->qp.rq.pc & (conn->qp.rq.size - 1); + data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix); + data->byte_count = cpu_to_be32(buf->sg[0].size); + data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); + data->addr = cpu_to_be64(buf->sg[0].dma_addr); + + conn->qp.rq.pc++; + conn->qp.rq.bufs[ix] = buf; + + /* Make sure that descriptors are written before doorbell record. */ + dma_wmb(); + *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff); +out: + return err; +} + +static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe) +{ + /* ensure wqe is visible to device before updating doorbell record */ + dma_wmb(); + *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc); + /* Make sure that doorbell record is visible before ringing */ + wmb(); + mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET, NULL); +} + +static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ + struct mlx5_wqe_ctrl_seg *ctrl; + struct mlx5_wqe_data_seg *data; + unsigned int ix, sgi; + int size = 1; + + ix = conn->qp.sq.pc & (conn->qp.sq.size - 1); + + ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix); + data = (void *)(ctrl + 1); + + for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) { + if (!buf->sg[sgi].data) + break; + data->byte_count = cpu_to_be32(buf->sg[sgi].size); + data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key); + data->addr = cpu_to_be64(buf->sg[sgi].dma_addr); + data++; + size++; + } + + ctrl->imm = 0; + ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) | + MLX5_OPCODE_SEND); + ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.mqp.qpn << 8)); + + conn->qp.sq.pc++; + conn->qp.sq.bufs[ix] = buf; + mlx5_fpga_conn_notify_hw(conn, ctrl); +} + +int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ + unsigned long flags; + int err; + + if (!conn->qp.active) + return -ENOTCONN; + + err = mlx5_fpga_conn_map_buf(conn, buf); + if (err) + return err; + + spin_lock_irqsave(&conn->qp.sq.lock, flags); + + if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) { + list_add_tail(&buf->list, &conn->qp.sq.backlog); + goto out_unlock; + } + + mlx5_fpga_conn_post_send(conn, buf); + +out_unlock: + spin_unlock_irqrestore(&conn->qp.sq.lock, flags); + return err; +} + +static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_dma_buf *buf; + int err; + + buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0); + if (!buf) + return -ENOMEM; + + buf->sg[0].data = (void *)(buf + 1); + buf->sg[0].size = MLX5_FPGA_RECV_SIZE; + buf->dma_dir = DMA_FROM_DEVICE; + + err = mlx5_fpga_conn_post_recv(conn, buf); + if (err) + kfree(buf); + + return err; +} + +static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn, + struct mlx5_core_mkey *mkey) +{ + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + void *mkc; + u32 *in; + int err; + + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); + MLX5_SET(mkc, mkc, lw, 1); + MLX5_SET(mkc, mkc, lr, 1); + + MLX5_SET(mkc, mkc, pd, pdn); + MLX5_SET(mkc, mkc, length64, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + + err = mlx5_core_create_mkey(mdev, mkey, in, inlen); + + kvfree(in); + return err; +} + +static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn, + struct mlx5_cqe64 *cqe, u8 status) +{ + struct mlx5_fpga_dma_buf *buf; + int ix, err; + + ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1); + buf = conn->qp.rq.bufs[ix]; + conn->qp.rq.bufs[ix] = NULL; + if (!status) + buf->sg[0].size = be32_to_cpu(cqe->byte_cnt); + conn->qp.rq.cc++; + + if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) + mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", + buf, conn->fpga_qpn, status); + else + mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n", + buf, conn->fpga_qpn, status); + + mlx5_fpga_conn_unmap_buf(conn, buf); + + if (unlikely(status || !conn->qp.active)) { + conn->qp.active = false; + kfree(buf); + return; + } + + mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n", + buf->sg[0].size); + conn->recv_cb(conn->cb_arg, buf); + + buf->sg[0].size = MLX5_FPGA_RECV_SIZE; + err = mlx5_fpga_conn_post_recv(conn, buf); + if (unlikely(err)) { + mlx5_fpga_warn(conn->fdev, + "Failed to re-post recv buf: %d\n", err); + kfree(buf); + } +} + +static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn, + struct mlx5_cqe64 *cqe, u8 status) +{ + struct mlx5_fpga_dma_buf *buf, *nextbuf; + unsigned long flags; + int ix; + + spin_lock_irqsave(&conn->qp.sq.lock, flags); + + ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1); + buf = conn->qp.sq.bufs[ix]; + conn->qp.sq.bufs[ix] = NULL; + conn->qp.sq.cc++; + + /* Handle backlog still under the spinlock to ensure message post order */ + if (unlikely(!list_empty(&conn->qp.sq.backlog))) { + if (likely(conn->qp.active)) { + nextbuf = list_first_entry(&conn->qp.sq.backlog, + struct mlx5_fpga_dma_buf, list); + list_del(&nextbuf->list); + mlx5_fpga_conn_post_send(conn, nextbuf); + } + } + + spin_unlock_irqrestore(&conn->qp.sq.lock, flags); + + if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR))) + mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", + buf, conn->fpga_qpn, status); + else + mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n", + buf, conn->fpga_qpn, status); + + mlx5_fpga_conn_unmap_buf(conn, buf); + + if (likely(buf->complete)) + buf->complete(conn, conn->fdev, buf, status); + + if (unlikely(status)) + conn->qp.active = false; +} + +static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn, + struct mlx5_cqe64 *cqe) +{ + u8 opcode, status = 0; + + opcode = cqe->op_own >> 4; + + switch (opcode) { + case MLX5_CQE_REQ_ERR: + status = ((struct mlx5_err_cqe *)cqe)->syndrome; + /* Fall through */ + case MLX5_CQE_REQ: + mlx5_fpga_conn_sq_cqe(conn, cqe, status); + break; + + case MLX5_CQE_RESP_ERR: + status = ((struct mlx5_err_cqe *)cqe)->syndrome; + /* Fall through */ + case MLX5_CQE_RESP_SEND: + mlx5_fpga_conn_rq_cqe(conn, cqe, status); + break; + default: + mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n", + opcode); + } +} + +static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn) +{ + mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT, + conn->fdev->conn_res.uar->map, conn->cq.wq.cc); +} + +static void mlx5_fpga_conn_cq_event(struct mlx5_core_cq *mcq, + enum mlx5_event event) +{ + struct mlx5_fpga_conn *conn; + + conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq); + mlx5_fpga_warn(conn->fdev, "CQ event %u on CQ #%u\n", event, mcq->cqn); +} + +static void mlx5_fpga_conn_event(struct mlx5_core_qp *mqp, int event) +{ + struct mlx5_fpga_conn *conn; + + conn = container_of(mqp, struct mlx5_fpga_conn, qp.mqp); + mlx5_fpga_warn(conn->fdev, "QP event %u on QP #%u\n", event, mqp->qpn); +} + +static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn, + unsigned int budget) +{ + struct mlx5_cqe64 *cqe; + + while (budget) { + cqe = mlx5_cqwq_get_cqe(&conn->cq.wq); + if (!cqe) + break; + + budget--; + mlx5_cqwq_pop(&conn->cq.wq); + mlx5_fpga_conn_handle_cqe(conn, cqe); + mlx5_cqwq_update_db_record(&conn->cq.wq); + } + if (!budget) { + tasklet_schedule(&conn->cq.tasklet); + return; + } + + mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc); + /* ensure cq space is freed before enabling more cqes */ + wmb(); + mlx5_fpga_conn_arm_cq(conn); +} + +static void mlx5_fpga_conn_cq_tasklet(unsigned long data) +{ + struct mlx5_fpga_conn *conn = (void *)data; + + if (unlikely(!conn->qp.active)) + return; + mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); +} + +static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq) +{ + struct mlx5_fpga_conn *conn; + + conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq); + if (unlikely(!conn->qp.active)) + return; + mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET); +} + +static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0}; + struct mlx5_wq_param wqp; + struct mlx5_cqe64 *cqe; + int inlen, err, eqn; + unsigned int irqn; + void *cqc, *in; + __be64 *pas; + u32 i; + + cq_size = roundup_pow_of_two(cq_size); + MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size)); + + wqp.buf_numa_node = mdev->priv.numa_node; + wqp.db_numa_node = mdev->priv.numa_node; + + err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq, + &conn->cq.wq_ctrl); + if (err) + return err; + + for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) { + cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i); + cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK; + } + + inlen = MLX5_ST_SZ_BYTES(create_cq_in) + + sizeof(u64) * conn->cq.wq_ctrl.frag_buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_cqwq; + } + + err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn); + if (err) + goto err_cqwq; + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size)); + MLX5_SET(cqc, cqc, c_eqn, eqn); + MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index); + MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.frag_buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma); + + pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas); + mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.frag_buf, pas); + + err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen); + kvfree(in); + + if (err) + goto err_cqwq; + + conn->cq.mcq.cqe_sz = 64; + conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db; + conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1; + *conn->cq.mcq.set_ci_db = 0; + *conn->cq.mcq.arm_db = 0; + conn->cq.mcq.vector = 0; + conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete; + conn->cq.mcq.event = mlx5_fpga_conn_cq_event; + conn->cq.mcq.irqn = irqn; + conn->cq.mcq.uar = fdev->conn_res.uar; + tasklet_init(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet, + (unsigned long)conn); + + mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn); + + goto out; + +err_cqwq: + mlx5_cqwq_destroy(&conn->cq.wq_ctrl); +out: + return err; +} + +static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn) +{ + tasklet_disable(&conn->cq.tasklet); + tasklet_kill(&conn->cq.tasklet); + mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq); + mlx5_cqwq_destroy(&conn->cq.wq_ctrl); +} + +static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + struct mlx5_wq_param wqp; + + wqp.buf_numa_node = mdev->priv.numa_node; + wqp.db_numa_node = mdev->priv.numa_node; + + return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq, + &conn->qp.wq_ctrl); +} + +static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn, + unsigned int tx_size, unsigned int rx_size) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {0}; + void *in = NULL, *qpc; + int err, inlen; + + conn->qp.rq.pc = 0; + conn->qp.rq.cc = 0; + conn->qp.rq.size = roundup_pow_of_two(rx_size); + conn->qp.sq.pc = 0; + conn->qp.sq.cc = 0; + conn->qp.sq.size = roundup_pow_of_two(tx_size); + + MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); + MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size)); + MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size)); + err = mlx5_fpga_conn_create_wq(conn, temp_qpc); + if (err) + goto out; + + conn->qp.rq.bufs = kvzalloc(sizeof(conn->qp.rq.bufs[0]) * + conn->qp.rq.size, GFP_KERNEL); + if (!conn->qp.rq.bufs) { + err = -ENOMEM; + goto err_wq; + } + + conn->qp.sq.bufs = kvzalloc(sizeof(conn->qp.sq.bufs[0]) * + conn->qp.sq.size, GFP_KERNEL); + if (!conn->qp.sq.bufs) { + err = -ENOMEM; + goto err_rq_bufs; + } + + inlen = MLX5_ST_SZ_BYTES(create_qp_in) + + MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * + conn->qp.wq_ctrl.buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_sq_bufs; + } + + qpc = MLX5_ADDR_OF(create_qp_in, in, qpc); + MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index); + MLX5_SET(qpc, qpc, log_page_size, + conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET(qpc, qpc, fre, 1); + MLX5_SET(qpc, qpc, rlky, 1); + MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); + MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); + MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn); + MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4); + MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size)); + MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ); + MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size)); + MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); + MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); + MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); + if (MLX5_CAP_GEN(mdev, cqe_version) == 1) + MLX5_SET(qpc, qpc, user_index, 0xFFFFFF); + + mlx5_fill_page_array(&conn->qp.wq_ctrl.buf, + (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas)); + + err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen); + if (err) + goto err_sq_bufs; + + conn->qp.mqp.event = mlx5_fpga_conn_event; + mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.mqp.qpn); + + goto out; + +err_sq_bufs: + kvfree(conn->qp.sq.bufs); +err_rq_bufs: + kvfree(conn->qp.rq.bufs); +err_wq: + mlx5_wq_destroy(&conn->qp.wq_ctrl); +out: + kvfree(in); + return err; +} + +static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn) +{ + int ix; + + for (ix = 0; ix < conn->qp.rq.size; ix++) { + if (!conn->qp.rq.bufs[ix]) + continue; + mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]); + kfree(conn->qp.rq.bufs[ix]); + conn->qp.rq.bufs[ix] = NULL; + } +} + +static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_dma_buf *buf, *temp; + int ix; + + for (ix = 0; ix < conn->qp.sq.size; ix++) { + buf = conn->qp.sq.bufs[ix]; + if (!buf) + continue; + conn->qp.sq.bufs[ix] = NULL; + mlx5_fpga_conn_unmap_buf(conn, buf); + if (!buf->complete) + continue; + buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); + } + list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) { + mlx5_fpga_conn_unmap_buf(conn, buf); + if (!buf->complete) + continue; + buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR); + } +} + +static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn) +{ + mlx5_core_destroy_qp(conn->fdev->mdev, &conn->qp.mqp); + mlx5_fpga_conn_free_recv_bufs(conn); + mlx5_fpga_conn_flush_send_bufs(conn); + kvfree(conn->qp.sq.bufs); + kvfree(conn->qp.rq.bufs); + mlx5_wq_destroy(&conn->qp.wq_ctrl); +} + +static inline int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn) +{ + struct mlx5_core_dev *mdev = conn->fdev->mdev; + + mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.mqp.qpn); + + return mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2RST_QP, 0, NULL, + &conn->qp.mqp); +} + +static inline int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + u32 *qpc = NULL; + int err; + + mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.mqp.qpn); + + qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); + if (!qpc) { + err = -ENOMEM; + goto out; + } + + MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC); + MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED); + MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); + MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM); + MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn); + MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn); + MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn); + MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma); + + err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, qpc, + &conn->qp.mqp); + if (err) { + mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); + goto out; + } + +out: + kfree(qpc); + return err; +} + +static inline int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + u32 *qpc = NULL; + int err; + + mlx5_fpga_dbg(conn->fdev, "QP RTR\n"); + + qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); + if (!qpc) { + err = -ENOMEM; + goto out; + } + + MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES); + MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg)); + MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn); + MLX5_SET(qpc, qpc, next_rcv_psn, + MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn)); + MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX); + MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM); + ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32), + MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32)); + MLX5_SET(qpc, qpc, primary_address_path.udp_sport, + MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port)); + MLX5_SET(qpc, qpc, primary_address_path.src_addr_index, + conn->qp.sgid_index); + MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0); + memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip), + MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip), + MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip)); + + err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, qpc, + &conn->qp.mqp); + if (err) { + mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); + goto out; + } + +out: + kfree(qpc); + return err; +} + +static inline int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + u32 *qpc = NULL; + u32 opt_mask; + int err; + + mlx5_fpga_dbg(conn->fdev, "QP RTS\n"); + + qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL); + if (!qpc) { + err = -ENOMEM; + goto out; + } + + MLX5_SET(qpc, qpc, log_ack_req_freq, 8); + MLX5_SET(qpc, qpc, min_rnr_nak, 0x12); + MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */ + MLX5_SET(qpc, qpc, next_send_psn, + MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn)); + MLX5_SET(qpc, qpc, retry_count, 7); + MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */ + + opt_mask = MLX5_QP_OPTPAR_RNR_TIMEOUT; + err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, opt_mask, qpc, + &conn->qp.mqp); + if (err) { + mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err); + goto out; + } + +out: + kfree(qpc); + return err; +} + +static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + int err; + + MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE); + err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, + MLX5_FPGA_QPC_STATE, &conn->fpga_qpc); + if (err) { + mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err); + goto out; + } + + err = mlx5_fpga_conn_reset_qp(conn); + if (err) { + mlx5_fpga_err(fdev, "Failed to change QP state to reset\n"); + goto err_fpga_qp; + } + + err = mlx5_fpga_conn_init_qp(conn); + if (err) { + mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n"); + goto err_fpga_qp; + } + conn->qp.active = true; + + while (!mlx5_fpga_conn_post_recv_buf(conn)) + ; + + err = mlx5_fpga_conn_rtr_qp(conn); + if (err) { + mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n"); + goto err_recv_bufs; + } + + err = mlx5_fpga_conn_rts_qp(conn); + if (err) { + mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n"); + goto err_recv_bufs; + } + goto out; + +err_recv_bufs: + mlx5_fpga_conn_free_recv_bufs(conn); +err_fpga_qp: + MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); + if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn, + MLX5_FPGA_QPC_STATE, &conn->fpga_qpc)) + mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n"); +out: + return err; +} + +struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_conn_attr *attr, + enum mlx5_ifc_fpga_qp_type qp_type) +{ + struct mlx5_fpga_conn *ret, *conn; + u8 *remote_mac, *remote_ip; + int err; + + if (!attr->recv_cb) + return ERR_PTR(-EINVAL); + + conn = kzalloc(sizeof(*conn), GFP_KERNEL); + if (!conn) + return ERR_PTR(-ENOMEM); + + conn->fdev = fdev; + INIT_LIST_HEAD(&conn->qp.sq.backlog); + + spin_lock_init(&conn->qp.sq.lock); + + conn->recv_cb = attr->recv_cb; + conn->cb_arg = attr->cb_arg; + + remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32); + err = mlx5_query_nic_vport_mac_address(fdev->mdev, 0, remote_mac); + if (err) { + mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err); + ret = ERR_PTR(err); + goto err; + } + + /* Build Modified EUI-64 IPv6 address from the MAC address */ + remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip); + remote_ip[0] = 0xfe; + remote_ip[1] = 0x80; + addrconf_addr_eui48(&remote_ip[8], remote_mac); + + err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index); + if (err) { + mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err); + ret = ERR_PTR(err); + goto err; + } + + err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, + MLX5_ROCE_VERSION_2, + MLX5_ROCE_L3_TYPE_IPV6, + remote_ip, remote_mac, true, 0); + if (err) { + mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err); + ret = ERR_PTR(err); + goto err_rsvd_gid; + } + mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index); + + /* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe, + * created during processing of the cqe + */ + err = mlx5_fpga_conn_create_cq(conn, + (attr->tx_size + attr->rx_size) * 2); + if (err) { + mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err); + ret = ERR_PTR(err); + goto err_gid; + } + + mlx5_fpga_conn_arm_cq(conn); + + err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size); + if (err) { + mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err); + ret = ERR_PTR(err); + goto err_cq; + } + + MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT); + MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type); + MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC); + MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q); + MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0); + MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1); + MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0); + MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY); + MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.mqp.qpn); + MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7); + MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7); + + err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc, + &conn->fpga_qpn); + if (err) { + mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err); + ret = ERR_PTR(err); + goto err_qp; + } + + err = mlx5_fpga_conn_connect(conn); + if (err) { + ret = ERR_PTR(err); + goto err_conn; + } + + mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn); + ret = conn; + goto out; + +err_conn: + mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); +err_qp: + mlx5_fpga_conn_destroy_qp(conn); +err_cq: + mlx5_fpga_conn_destroy_cq(conn); +err_gid: + mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL, + NULL, false, 0); +err_rsvd_gid: + mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index); +err: + kfree(conn); +out: + return ret; +} + +void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn) +{ + struct mlx5_fpga_device *fdev = conn->fdev; + struct mlx5_core_dev *mdev = fdev->mdev; + int err = 0; + + conn->qp.active = false; + tasklet_disable(&conn->cq.tasklet); + synchronize_irq(conn->cq.mcq.irqn); + + mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn); + err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2ERR_QP, 0, NULL, + &conn->qp.mqp); + if (err) + mlx5_fpga_warn(fdev, "qp_modify 2ERR failed: %d\n", err); + mlx5_fpga_conn_destroy_qp(conn); + mlx5_fpga_conn_destroy_cq(conn); + + mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0, + NULL, NULL, false, 0); + mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index); + kfree(conn); +} + +int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev) +{ + int err; + + err = mlx5_nic_vport_enable_roce(fdev->mdev); + if (err) { + mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err); + goto out; + } + + fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev); + if (IS_ERR(fdev->conn_res.uar)) { + err = PTR_ERR(fdev->conn_res.uar); + mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err); + goto err_roce; + } + mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n", + fdev->conn_res.uar->index); + + err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn); + if (err) { + mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err); + goto err_uar; + } + mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn); + + err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn, + &fdev->conn_res.mkey); + if (err) { + mlx5_fpga_err(fdev, "create mkey failed, %d\n", err); + goto err_dealloc_pd; + } + mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key); + + return 0; + +err_dealloc_pd: + mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); +err_uar: + mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); +err_roce: + mlx5_nic_vport_disable_roce(fdev->mdev); +out: + return err; +} + +void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev) +{ + mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey); + mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn); + mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar); + mlx5_nic_vport_disable_roce(fdev->mdev); +} diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_core.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_core.c new file mode 100644 index 000000000000..9805d0f58f5e --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_core.c @@ -0,0 +1,568 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +static LIST_HEAD(mlx5_fpga_devices); +static LIST_HEAD(mlx5_fpga_clients); +/* protects access between client un/registration and device add/remove calls */ +static DEFINE_MUTEX(mlx5_fpga_mutex); + +static const char *const mlx5_fpga_error_strings[] = { + "Null Syndrome", + "Corrupted DDR", + "Flash Timeout", + "Internal Link Error", + "Watchdog HW Failure", + "I2C Failure", + "Image Changed", + "Temperature Critical", +}; + +static const char * const mlx5_fpga_qp_error_strings[] = { + "Null Syndrome", + "Retry Counter Expired", + "RNR Expired", +}; + +static void client_context_destroy(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client_data *context) +{ + mlx5_fpga_dbg(fdev, "Deleting client context %p of client %p\n", + context, context->client); + if (context->client->destroy) + context->client->destroy(fdev); + list_del(&context->list); + kfree(context); +} + +static int client_context_create(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client *client, + struct mlx5_fpga_client_data **pctx) +{ + struct mlx5_fpga_client_data *context; + + context = kmalloc(sizeof(*context), GFP_KERNEL); + if (!context) + return -ENOMEM; + + context->client = client; + context->data = NULL; + context->added = false; + list_add(&context->list, &fdev->client_data_list); + + mlx5_fpga_dbg(fdev, "Adding client context %p client %p\n", + context, client); + + if (client->create) + client->create(fdev); + + if (pctx) + *pctx = context; + return 0; +} + +static struct mlx5_fpga_device *mlx5_fpga_device_alloc(void) +{ + struct mlx5_fpga_device *fdev = NULL; + + fdev = kzalloc(sizeof(*fdev), GFP_KERNEL); + if (!fdev) + return NULL; + + spin_lock_init(&fdev->state_lock); + init_completion(&fdev->load_event); + fdev->fdev_state = MLX5_FDEV_STATE_NONE; + INIT_LIST_HEAD(&fdev->client_data_list); + return fdev; +} + +static const char *mlx5_fpga_image_name(enum mlx5_fpga_image image) +{ + switch (image) { + case MLX5_FPGA_IMAGE_USER: + return "user"; + case MLX5_FPGA_IMAGE_FACTORY: + return "factory"; + default: + return "unknown"; + } +} + +static const char *mlx5_fpga_name(u32 fpga_id) +{ + static char ret[32]; + + switch (fpga_id) { + case MLX5_FPGA_NEWTON: + return "Newton"; + case MLX5_FPGA_EDISON: + return "Edison"; + case MLX5_FPGA_MORSE: + return "Morse"; + } + + snprintf(ret, sizeof(ret), "Unknown %d", fpga_id); + return ret; +} + +static int mlx5_fpga_device_load_check(struct mlx5_fpga_device *fdev) +{ + struct mlx5_fpga_query query; + int err; + + err = mlx5_fpga_query(fdev->mdev, &query); + if (err) { + mlx5_fpga_err(fdev, "Failed to query status: %d\n", err); + return err; + } + + fdev->last_admin_image = query.admin_image; + fdev->last_oper_image = query.oper_image; + fdev->image_status = query.image_status; + + mlx5_fpga_info(fdev, "Status %u; Admin image %u; Oper image %u\n", + query.image_status, query.admin_image, query.oper_image); + + /* For Morse project FPGA has no influence to network functionality */ + if (MLX5_CAP_FPGA(fdev->mdev, fpga_id) == MLX5_FPGA_MORSE) + return 0; + + if (query.image_status != MLX5_FPGA_STATUS_SUCCESS) { + mlx5_fpga_err(fdev, "%s image failed to load; status %u\n", + mlx5_fpga_image_name(fdev->last_oper_image), + query.image_status); + return -EIO; + } + + return 0; +} + +static int mlx5_fpga_device_brb(struct mlx5_fpga_device *fdev) +{ + int err; + struct mlx5_core_dev *mdev = fdev->mdev; + + err = mlx5_fpga_ctrl_op(mdev, MLX5_FPGA_CTRL_OPERATION_SANDBOX_BYPASS_ON); + if (err) { + mlx5_fpga_err(fdev, "Failed to set bypass on: %d\n", err); + return err; + } + err = mlx5_fpga_ctrl_op(mdev, MLX5_FPGA_CTRL_OPERATION_RESET_SANDBOX); + if (err) { + mlx5_fpga_err(fdev, "Failed to reset SBU: %d\n", err); + return err; + } + err = mlx5_fpga_ctrl_op(mdev, MLX5_FPGA_CTRL_OPERATION_SANDBOX_BYPASS_OFF); + if (err) { + mlx5_fpga_err(fdev, "Failed to set bypass off: %d\n", err); + return err; + } + return 0; +} + +int mlx5_fpga_device_start(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_client_data *client_context; + struct mlx5_fpga_device *fdev = mdev->fpga; + struct mlx5_fpga_conn_attr conn_attr = {0}; + struct mlx5_fpga_conn *conn; + unsigned int max_num_qps; + unsigned long flags; + u32 fpga_id; + u32 vid; + u16 pid; + int err; + + if (!fdev) + return 0; + + err = mlx5_fpga_caps(fdev->mdev); + if (err) + goto out; + + err = mlx5_fpga_device_load_check(fdev); + if (err) + goto out; + + fpga_id = MLX5_CAP_FPGA(fdev->mdev, fpga_id); + mlx5_fpga_info(fdev, "FPGA card %s\n", mlx5_fpga_name(fpga_id)); + + if (fpga_id == MLX5_FPGA_MORSE) + goto out; + + mlx5_fpga_info(fdev, "%s(%d) image, version %u; SBU %06x:%04x version %d\n", + mlx5_fpga_image_name(fdev->last_oper_image), + fdev->last_oper_image, + MLX5_CAP_FPGA(fdev->mdev, image_version), + MLX5_CAP_FPGA(fdev->mdev, ieee_vendor_id), + MLX5_CAP_FPGA(fdev->mdev, sandbox_product_id), + MLX5_CAP_FPGA(fdev->mdev, sandbox_product_version)); + + max_num_qps = MLX5_CAP_FPGA(mdev, shell_caps.max_num_qps); + err = mlx5_core_reserve_gids(mdev, max_num_qps); + if (err) + goto out; + +#ifdef NOT_YET + /* XXXKIB */ + err = mlx5_fpga_conn_device_init(fdev); +#else + err = 0; +#endif + if (err) + goto err_rsvd_gid; + + err = mlx5_fpga_trans_device_init(fdev); + if (err) { + mlx5_fpga_err(fdev, "Failed to init transaction: %d\n", + err); + goto err_conn_init; + } + + conn_attr.tx_size = MLX5_FPGA_TID_COUNT; + conn_attr.rx_size = MLX5_FPGA_TID_COUNT; + conn_attr.recv_cb = mlx5_fpga_trans_recv; + conn_attr.cb_arg = fdev; +#ifdef NOT_YET + /* XXXKIB */ + conn = mlx5_fpga_conn_create(fdev, &conn_attr, + MLX5_FPGA_QPC_QP_TYPE_SHELL_QP); + if (IS_ERR(conn)) { + err = PTR_ERR(conn); + mlx5_fpga_err(fdev, "Failed to create shell conn: %d\n", err); + goto err_trans; + } +#else + conn = NULL; +#endif + fdev->shell_conn = conn; + + if (fdev->last_oper_image == MLX5_FPGA_IMAGE_USER) { + err = mlx5_fpga_device_brb(fdev); + if (err) + goto err_shell_conn; + + vid = MLX5_CAP_FPGA(fdev->mdev, ieee_vendor_id); + pid = MLX5_CAP_FPGA(fdev->mdev, sandbox_product_id); + mutex_lock(&mlx5_fpga_mutex); + list_for_each_entry(client_context, &fdev->client_data_list, + list) { + if (client_context->client->add(fdev, vid, pid)) + continue; + client_context->added = true; + } + mutex_unlock(&mlx5_fpga_mutex); + } + + goto out; + +err_shell_conn: + if (fdev->shell_conn) { +#ifdef NOT_YET + /* XXXKIB */ + mlx5_fpga_conn_destroy(fdev->shell_conn); +#endif + fdev->shell_conn = NULL; + } + +#ifdef NOT_YET + /* XXXKIB */ +err_trans: +#endif + mlx5_fpga_trans_device_cleanup(fdev); + +err_conn_init: +#ifdef NOT_YET + /* XXXKIB */ + mlx5_fpga_conn_device_cleanup(fdev); +#endif + +err_rsvd_gid: + mlx5_core_unreserve_gids(mdev, max_num_qps); +out: + spin_lock_irqsave(&fdev->state_lock, flags); + fdev->fdev_state = err ? MLX5_FDEV_STATE_FAILURE : MLX5_FDEV_STATE_SUCCESS; + spin_unlock_irqrestore(&fdev->state_lock, flags); + return err; +} + +int mlx5_fpga_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_device *fdev = NULL; + struct mlx5_fpga_client *client; + + if (!MLX5_CAP_GEN(mdev, fpga)) { + mlx5_core_dbg(mdev, "FPGA capability not present\n"); + return 0; + } + + mlx5_core_dbg(mdev, "Initializing FPGA\n"); + + fdev = mlx5_fpga_device_alloc(); + if (!fdev) + return -ENOMEM; + + fdev->mdev = mdev; + mdev->fpga = fdev; + + mutex_lock(&mlx5_fpga_mutex); + + list_add_tail(&fdev->list, &mlx5_fpga_devices); + list_for_each_entry(client, &mlx5_fpga_clients, list) + client_context_create(fdev, client, NULL); + + mutex_unlock(&mlx5_fpga_mutex); + return 0; +} + +void mlx5_fpga_device_stop(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_client_data *client_context; + struct mlx5_fpga_device *fdev = mdev->fpga; + unsigned int max_num_qps; + unsigned long flags; + int err; + + if (!fdev) + return; + + spin_lock_irqsave(&fdev->state_lock, flags); + if (MLX5_CAP_FPGA(mdev, fpga_id) == MLX5_FPGA_MORSE) + return; + + if (fdev->fdev_state != MLX5_FDEV_STATE_SUCCESS) { + spin_unlock_irqrestore(&fdev->state_lock, flags); + return; + } + fdev->fdev_state = MLX5_FDEV_STATE_NONE; + spin_unlock_irqrestore(&fdev->state_lock, flags); + + if (fdev->last_oper_image == MLX5_FPGA_IMAGE_USER) { + err = mlx5_fpga_ctrl_op(mdev, MLX5_FPGA_CTRL_OPERATION_SANDBOX_BYPASS_ON); + if (err) + mlx5_fpga_err(fdev, "Failed to re-set SBU bypass on: %d\n", + err); + } + + mutex_lock(&mlx5_fpga_mutex); + list_for_each_entry(client_context, &fdev->client_data_list, list) { + if (!client_context->added) + continue; + client_context->client->remove(fdev); + client_context->added = false; + } + mutex_unlock(&mlx5_fpga_mutex); + + if (fdev->shell_conn) { +#ifdef NOT_YET + /* XXXKIB */ + mlx5_fpga_conn_destroy(fdev->shell_conn); +#endif + fdev->shell_conn = NULL; + mlx5_fpga_trans_device_cleanup(fdev); + } +#ifdef NOT_YET + /* XXXKIB */ + mlx5_fpga_conn_device_cleanup(fdev); +#endif + max_num_qps = MLX5_CAP_FPGA(mdev, shell_caps.max_num_qps); + mlx5_core_unreserve_gids(mdev, max_num_qps); +} + +void mlx5_fpga_cleanup(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_client_data *context, *tmp; + struct mlx5_fpga_device *fdev = mdev->fpga; + + if (!fdev) + return; + + mutex_lock(&mlx5_fpga_mutex); + + mlx5_fpga_device_stop(mdev); + + list_for_each_entry_safe(context, tmp, &fdev->client_data_list, list) + client_context_destroy(fdev, context); + + list_del(&fdev->list); + kfree(fdev); + mdev->fpga = NULL; + + mutex_unlock(&mlx5_fpga_mutex); +} + +static const char *mlx5_fpga_syndrome_to_string(u8 syndrome) +{ + if (syndrome < ARRAY_SIZE(mlx5_fpga_error_strings)) + return mlx5_fpga_error_strings[syndrome]; + return "Unknown"; +} + +static const char *mlx5_fpga_qp_syndrome_to_string(u8 syndrome) +{ + if (syndrome < ARRAY_SIZE(mlx5_fpga_qp_error_strings)) + return mlx5_fpga_qp_error_strings[syndrome]; + return "Unknown"; +} + +void mlx5_fpga_event(struct mlx5_core_dev *mdev, u8 event, void *data) +{ + struct mlx5_fpga_device *fdev = mdev->fpga; + const char *event_name; + bool teardown = false; + unsigned long flags; + u32 fpga_qpn; + u8 syndrome; + + switch (event) { + case MLX5_EVENT_TYPE_FPGA_ERROR: + syndrome = MLX5_GET(fpga_error_event, data, syndrome); + event_name = mlx5_fpga_syndrome_to_string(syndrome); + break; + case MLX5_EVENT_TYPE_FPGA_QP_ERROR: + syndrome = MLX5_GET(fpga_qp_error_event, data, syndrome); + event_name = mlx5_fpga_qp_syndrome_to_string(syndrome); + fpga_qpn = MLX5_GET(fpga_qp_error_event, data, fpga_qpn); + mlx5_fpga_err(fdev, "Error %u on QP %u: %s\n", + syndrome, fpga_qpn, event_name); + break; + default: + mlx5_fpga_warn_ratelimited(fdev, "Unexpected event %u\n", + event); + return; + } + + spin_lock_irqsave(&fdev->state_lock, flags); + switch (fdev->fdev_state) { + case MLX5_FDEV_STATE_SUCCESS: + mlx5_fpga_warn(fdev, "Error %u: %s\n", syndrome, event_name); + teardown = true; + break; + case MLX5_FDEV_STATE_IN_PROGRESS: + if (syndrome != MLX5_FPGA_ERROR_EVENT_SYNDROME_IMAGE_CHANGED) + mlx5_fpga_warn(fdev, "Error while loading %u: %s\n", + syndrome, event_name); + complete(&fdev->load_event); + break; + default: + mlx5_fpga_warn_ratelimited(fdev, "Unexpected error event %u: %s\n", + syndrome, event_name); + } + spin_unlock_irqrestore(&fdev->state_lock, flags); + /* We tear-down the card's interfaces and functionality because + * the FPGA bump-on-the-wire is misbehaving and we lose ability + * to communicate with the network. User may still be able to + * recover by re-programming or debugging the FPGA + */ + if (teardown) + mlx5_trigger_health_work(fdev->mdev); +} + +void mlx5_fpga_client_register(struct mlx5_fpga_client *client) +{ + struct mlx5_fpga_client_data *context; + struct mlx5_fpga_device *fdev; + bool call_add = false; + unsigned long flags; + u32 vid; + u16 pid; + int err; + + pr_debug("Client register %s\n", client->name); + + mutex_lock(&mlx5_fpga_mutex); + + list_add_tail(&client->list, &mlx5_fpga_clients); + + list_for_each_entry(fdev, &mlx5_fpga_devices, list) { + err = client_context_create(fdev, client, &context); + if (err) + continue; + + spin_lock_irqsave(&fdev->state_lock, flags); + call_add = (fdev->fdev_state == MLX5_FDEV_STATE_SUCCESS); + spin_unlock_irqrestore(&fdev->state_lock, flags); + + if (call_add) { + vid = MLX5_CAP_FPGA(fdev->mdev, ieee_vendor_id); + pid = MLX5_CAP_FPGA(fdev->mdev, sandbox_product_id); + if (!client->add(fdev, vid, pid)) + context->added = true; + } + } + + mutex_unlock(&mlx5_fpga_mutex); +} +EXPORT_SYMBOL(mlx5_fpga_client_register); + +void mlx5_fpga_client_unregister(struct mlx5_fpga_client *client) +{ + struct mlx5_fpga_client_data *context, *tmp_context; + struct mlx5_fpga_device *fdev; + + pr_debug("Client unregister %s\n", client->name); + + mutex_lock(&mlx5_fpga_mutex); + + list_for_each_entry(fdev, &mlx5_fpga_devices, list) { + list_for_each_entry_safe(context, tmp_context, + &fdev->client_data_list, + list) { + if (context->client != client) + continue; + if (context->added) + client->remove(fdev); + client_context_destroy(fdev, context); + break; + } + } + + list_del(&client->list); + mutex_unlock(&mlx5_fpga_mutex); +} +EXPORT_SYMBOL(mlx5_fpga_client_unregister); + +#if (__FreeBSD_version >= 1100000) +MODULE_DEPEND(mlx5fpga, linuxkpi, 1, 1, 1); +#endif +MODULE_DEPEND(mlx5fpga, mlx5, 1, 1, 1); +MODULE_VERSION(mlx5fpga, 1); diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_ipsec.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_ipsec.c new file mode 100644 index 000000000000..65f5984060ec --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_ipsec.c @@ -0,0 +1,377 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include + +#include +#include +#include +#include + +#define SBU_QP_QUEUE_SIZE 8 + +enum mlx5_ipsec_response_syndrome { + MLX5_IPSEC_RESPONSE_SUCCESS = 0, + MLX5_IPSEC_RESPONSE_ILLEGAL_REQUEST = 1, + MLX5_IPSEC_RESPONSE_SADB_ISSUE = 2, + MLX5_IPSEC_RESPONSE_WRITE_RESPONSE_ISSUE = 3, +}; + +enum mlx5_fpga_ipsec_sacmd_status { + MLX5_FPGA_IPSEC_SACMD_PENDING, + MLX5_FPGA_IPSEC_SACMD_SEND_FAIL, + MLX5_FPGA_IPSEC_SACMD_COMPLETE, +}; + +struct mlx5_ipsec_command_context { + struct mlx5_fpga_dma_buf buf; + struct mlx5_accel_ipsec_sa sa; + enum mlx5_fpga_ipsec_sacmd_status status; + int status_code; + struct completion complete; + struct mlx5_fpga_device *dev; + struct list_head list; /* Item in pending_cmds */ +}; + +struct mlx5_ipsec_sadb_resp { + __be32 syndrome; + __be32 sw_sa_handle; + u8 reserved[24]; +} __packed; + +struct mlx5_fpga_ipsec { + struct list_head pending_cmds; + spinlock_t pending_cmds_lock; /* Protects pending_cmds */ + u32 caps[MLX5_ST_SZ_DW(ipsec_extended_cap)]; + struct mlx5_fpga_conn *conn; +}; + +static bool mlx5_fpga_is_ipsec_device(struct mlx5_core_dev *mdev) +{ + if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga)) + return false; + + if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) != + MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX) + return false; + + if (MLX5_CAP_FPGA(mdev, sandbox_product_id) != + MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC) + return false; + + return true; +} + +static void mlx5_fpga_ipsec_send_complete(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_device *fdev, + struct mlx5_fpga_dma_buf *buf, + u8 status) +{ + struct mlx5_ipsec_command_context *context; + + if (status) { + context = container_of(buf, struct mlx5_ipsec_command_context, + buf); + mlx5_fpga_warn(fdev, "IPSec command send failed with status %u\n", + status); + context->status = MLX5_FPGA_IPSEC_SACMD_SEND_FAIL; + complete(&context->complete); + } +} + +static inline int syndrome_to_errno(enum mlx5_ipsec_response_syndrome syndrome) +{ + switch (syndrome) { + case MLX5_IPSEC_RESPONSE_SUCCESS: + return 0; + case MLX5_IPSEC_RESPONSE_SADB_ISSUE: + return -EEXIST; + case MLX5_IPSEC_RESPONSE_ILLEGAL_REQUEST: + return -EINVAL; + case MLX5_IPSEC_RESPONSE_WRITE_RESPONSE_ISSUE: + return -EIO; + } + return -EIO; +} + +static void mlx5_fpga_ipsec_recv(void *cb_arg, struct mlx5_fpga_dma_buf *buf) +{ + struct mlx5_ipsec_sadb_resp *resp = buf->sg[0].data; + struct mlx5_ipsec_command_context *context; + enum mlx5_ipsec_response_syndrome syndrome; + struct mlx5_fpga_device *fdev = cb_arg; + unsigned long flags; + + if (buf->sg[0].size < sizeof(*resp)) { + mlx5_fpga_warn(fdev, "Short receive from FPGA IPSec: %u < %zu bytes\n", + buf->sg[0].size, sizeof(*resp)); + return; + } + + mlx5_fpga_dbg(fdev, "mlx5_ipsec recv_cb syndrome %08x sa_id %x\n", + ntohl(resp->syndrome), ntohl(resp->sw_sa_handle)); + + spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags); + context = list_first_entry_or_null(&fdev->ipsec->pending_cmds, + struct mlx5_ipsec_command_context, + list); + if (context) + list_del(&context->list); + spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags); + + if (!context) { + mlx5_fpga_warn(fdev, "Received IPSec offload response without pending command request\n"); + return; + } + mlx5_fpga_dbg(fdev, "Handling response for %p\n", context); + + if (context->sa.sw_sa_handle != resp->sw_sa_handle) { + mlx5_fpga_err(fdev, "mismatch SA handle. cmd 0x%08x vs resp 0x%08x\n", + ntohl(context->sa.sw_sa_handle), + ntohl(resp->sw_sa_handle)); + return; + } + + syndrome = ntohl(resp->syndrome); + context->status_code = syndrome_to_errno(syndrome); + context->status = MLX5_FPGA_IPSEC_SACMD_COMPLETE; + + if (context->status_code) + mlx5_fpga_warn(fdev, "IPSec SADB command failed with syndrome %08x\n", + syndrome); + complete(&context->complete); +} + +void *mlx5_fpga_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev, + struct mlx5_accel_ipsec_sa *cmd) +{ + struct mlx5_ipsec_command_context *context; + struct mlx5_fpga_device *fdev = mdev->fpga; + unsigned long flags; + int res = 0; + + BUILD_BUG_ON((sizeof(struct mlx5_accel_ipsec_sa) & 3) != 0); + if (!fdev || !fdev->ipsec) + return ERR_PTR(-EOPNOTSUPP); + + context = kzalloc(sizeof(*context), GFP_ATOMIC); + if (!context) + return ERR_PTR(-ENOMEM); + + memcpy(&context->sa, cmd, sizeof(*cmd)); + context->buf.complete = mlx5_fpga_ipsec_send_complete; + context->buf.sg[0].size = sizeof(context->sa); + context->buf.sg[0].data = &context->sa; + init_completion(&context->complete); + context->dev = fdev; + spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags); + list_add_tail(&context->list, &fdev->ipsec->pending_cmds); + spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags); + + context->status = MLX5_FPGA_IPSEC_SACMD_PENDING; + + res = mlx5_fpga_sbu_conn_sendmsg(fdev->ipsec->conn, &context->buf); + if (res) { + mlx5_fpga_warn(fdev, "Failure sending IPSec command: %d\n", + res); + spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags); + list_del(&context->list); + spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags); + kfree(context); + return ERR_PTR(res); + } + /* Context will be freed by wait func after completion */ + return context; +} + +int mlx5_fpga_ipsec_sa_cmd_wait(void *ctx) +{ + struct mlx5_ipsec_command_context *context = ctx; + int res; + + res = wait_for_completion/*_killable XXXKIB*/(&context->complete); + if (res) { + mlx5_fpga_warn(context->dev, "Failure waiting for IPSec command response\n"); + return -EINTR; + } + + if (context->status == MLX5_FPGA_IPSEC_SACMD_COMPLETE) + res = context->status_code; + else + res = -EIO; + + kfree(context); + return res; +} + +u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_device *fdev = mdev->fpga; + u32 ret = 0; + + if (mlx5_fpga_is_ipsec_device(mdev)) + ret |= MLX5_ACCEL_IPSEC_DEVICE; + else + return ret; + + if (!fdev->ipsec) + return ret; + + if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, esp)) + ret |= MLX5_ACCEL_IPSEC_ESP; + + if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, ipv6)) + ret |= MLX5_ACCEL_IPSEC_IPV6; + + if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, lso)) + ret |= MLX5_ACCEL_IPSEC_LSO; + + return ret; +} + +unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_device *fdev = mdev->fpga; + + if (!fdev || !fdev->ipsec) + return 0; + + return MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, + number_of_ipsec_counters); +} + +int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters, + unsigned int counters_count) +{ + struct mlx5_fpga_device *fdev = mdev->fpga; + unsigned int i; + __be32 *data; + u32 count; + u64 addr; + int ret; + + if (!fdev || !fdev->ipsec) + return 0; + + addr = (u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, + ipsec_counters_addr_low) + + ((u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, + ipsec_counters_addr_high) << 32); + + count = mlx5_fpga_ipsec_counters_count(mdev); + + data = kzalloc(sizeof(*data) * count * 2, GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + goto out; + } + + ret = mlx5_fpga_mem_read(fdev, count * sizeof(u64), addr, data, + MLX5_FPGA_ACCESS_TYPE_DONTCARE); + if (ret < 0) { + mlx5_fpga_err(fdev, "Failed to read IPSec counters from HW: %d\n", + ret); + goto out; + } + ret = 0; + + if (count > counters_count) + count = counters_count; + + /* Each counter is low word, then high. But each word is big-endian */ + for (i = 0; i < count; i++) + counters[i] = (u64)ntohl(data[i * 2]) | + ((u64)ntohl(data[i * 2 + 1]) << 32); + +out: + kfree(data); + return ret; +} + +int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_conn_attr init_attr = {0}; + struct mlx5_fpga_device *fdev = mdev->fpga; + struct mlx5_fpga_conn *conn; + int err; + + if (!mlx5_fpga_is_ipsec_device(mdev)) + return 0; + + fdev->ipsec = kzalloc(sizeof(*fdev->ipsec), GFP_KERNEL); + if (!fdev->ipsec) + return -ENOMEM; + + err = mlx5_fpga_get_sbu_caps(fdev, sizeof(fdev->ipsec->caps), + fdev->ipsec->caps); + if (err) { + mlx5_fpga_err(fdev, "Failed to retrieve IPSec extended capabilities: %d\n", + err); + goto error; + } + + INIT_LIST_HEAD(&fdev->ipsec->pending_cmds); + spin_lock_init(&fdev->ipsec->pending_cmds_lock); + + init_attr.rx_size = SBU_QP_QUEUE_SIZE; + init_attr.tx_size = SBU_QP_QUEUE_SIZE; + init_attr.recv_cb = mlx5_fpga_ipsec_recv; + init_attr.cb_arg = fdev; + conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr); + if (IS_ERR(conn)) { + err = PTR_ERR(conn); + mlx5_fpga_err(fdev, "Error creating IPSec command connection %d\n", + err); + goto error; + } + fdev->ipsec->conn = conn; + return 0; + +error: + kfree(fdev->ipsec); + fdev->ipsec = NULL; + return err; +} + +void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev) +{ + struct mlx5_fpga_device *fdev = mdev->fpga; + + if (!mlx5_fpga_is_ipsec_device(mdev)) + return; + + mlx5_fpga_sbu_conn_destroy(fdev->ipsec->conn); + kfree(fdev->ipsec); + fdev->ipsec = NULL; +} diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_sdk.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_sdk.c new file mode 100644 index 000000000000..0a611f1e8388 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_sdk.c @@ -0,0 +1,459 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +/* #include "accel/ipsec.h" */ + +#define MLX5_FPGA_LOAD_TIMEOUT 25000 /* msec */ + +struct mem_transfer { + struct mlx5_fpga_transaction t; + struct completion comp; + u8 status; +}; + +struct mlx5_fpga_conn * +mlx5_fpga_sbu_conn_create(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_conn_attr *attr) +{ +#ifdef NOT_YET + /* XXXKIB */ + return mlx5_fpga_conn_create(fdev, attr, MLX5_FPGA_QPC_QP_TYPE_SANDBOX_QP); +#else + return (NULL); +#endif +} +EXPORT_SYMBOL(mlx5_fpga_sbu_conn_create); + +void mlx5_fpga_sbu_conn_destroy(struct mlx5_fpga_conn *conn) +{ +#ifdef NOT_YET + /* XXXKIB */ + mlx5_fpga_conn_destroy(conn); +#endif +} +EXPORT_SYMBOL(mlx5_fpga_sbu_conn_destroy); + +int mlx5_fpga_sbu_conn_sendmsg(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf) +{ +#ifdef NOT_YET + /* XXXKIB */ + return mlx5_fpga_conn_send(conn, buf); +#else + return (0); +#endif +} +EXPORT_SYMBOL(mlx5_fpga_sbu_conn_sendmsg); + +static void mem_complete(const struct mlx5_fpga_transaction *complete, + u8 status) +{ + struct mem_transfer *xfer; + + mlx5_fpga_dbg(complete->conn->fdev, + "transaction %p complete status %u", complete, status); + + xfer = container_of(complete, struct mem_transfer, t); + xfer->status = status; + complete_all(&xfer->comp); +} + +static int mem_transaction(struct mlx5_fpga_device *fdev, size_t size, u64 addr, + void *buf, enum mlx5_fpga_direction direction) +{ + int ret; + struct mem_transfer xfer; + + if (!fdev->shell_conn) { + ret = -ENOTCONN; + goto out; + } + + xfer.t.data = buf; + xfer.t.size = size; + xfer.t.addr = addr; + xfer.t.conn = fdev->shell_conn; + xfer.t.direction = direction; + xfer.t.complete1 = mem_complete; + init_completion(&xfer.comp); + ret = mlx5_fpga_xfer_exec(&xfer.t); + if (ret) { + mlx5_fpga_dbg(fdev, "Transfer execution failed: %d\n", ret); + goto out; + } + wait_for_completion(&xfer.comp); + if (xfer.status != 0) + ret = -EIO; + +out: + return ret; +} + +static int mlx5_fpga_mem_read_i2c(struct mlx5_fpga_device *fdev, size_t size, + u64 addr, u8 *buf) +{ + size_t max_size = MLX5_FPGA_ACCESS_REG_SIZE_MAX; + size_t bytes_done = 0; + u8 actual_size; + int err = 0; + + if (!size) + return -EINVAL; + + if (!fdev->mdev) + return -ENOTCONN; + + while (bytes_done < size) { + actual_size = min(max_size, (size - bytes_done)); + + err = mlx5_fpga_access_reg(fdev->mdev, actual_size, + addr + bytes_done, + buf + bytes_done, false); + if (err) { + mlx5_fpga_err(fdev, "Failed to read over I2C: %d\n", + err); + break; + } + + bytes_done += actual_size; + } + + return err; +} + +static int mlx5_fpga_mem_write_i2c(struct mlx5_fpga_device *fdev, size_t size, + u64 addr, u8 *buf) +{ + size_t max_size = MLX5_FPGA_ACCESS_REG_SIZE_MAX; + size_t bytes_done = 0; + u8 actual_size; + int err = 0; + + if (!size) + return -EINVAL; + + if (!fdev->mdev) + return -ENOTCONN; + + while (bytes_done < size) { + actual_size = min(max_size, (size - bytes_done)); + + err = mlx5_fpga_access_reg(fdev->mdev, actual_size, + addr + bytes_done, + buf + bytes_done, true); + if (err) { + mlx5_fpga_err(fdev, "Failed to write FPGA crspace\n"); + break; + } + + bytes_done += actual_size; + } + + return err; +} + +int mlx5_fpga_mem_read(struct mlx5_fpga_device *fdev, size_t size, u64 addr, + void *buf, enum mlx5_fpga_access_type access_type) +{ + int ret; + + if (access_type == MLX5_FPGA_ACCESS_TYPE_DONTCARE) + access_type = fdev->shell_conn ? MLX5_FPGA_ACCESS_TYPE_RDMA : + MLX5_FPGA_ACCESS_TYPE_I2C; + + mlx5_fpga_dbg(fdev, "Reading %zu bytes at 0x%jx over %s", + size, (uintmax_t)addr, access_type ? "RDMA" : "I2C"); + + switch (access_type) { + case MLX5_FPGA_ACCESS_TYPE_RDMA: + ret = mem_transaction(fdev, size, addr, buf, MLX5_FPGA_READ); + if (ret) + return ret; + break; + case MLX5_FPGA_ACCESS_TYPE_I2C: + ret = mlx5_fpga_mem_read_i2c(fdev, size, addr, buf); + if (ret) + return ret; + break; + default: + mlx5_fpga_warn(fdev, "Unexpected read access_type %u\n", + access_type); + return -EACCES; + } + + return size; +} +EXPORT_SYMBOL(mlx5_fpga_mem_read); + +int mlx5_fpga_mem_write(struct mlx5_fpga_device *fdev, size_t size, u64 addr, + void *buf, enum mlx5_fpga_access_type access_type) +{ + int ret; + + if (access_type == MLX5_FPGA_ACCESS_TYPE_DONTCARE) + access_type = fdev->shell_conn ? MLX5_FPGA_ACCESS_TYPE_RDMA : + MLX5_FPGA_ACCESS_TYPE_I2C; + + mlx5_fpga_dbg(fdev, "Writing %zu bytes at 0x%jx over %s", + size, (uintmax_t)addr, access_type ? "RDMA" : "I2C"); + + switch (access_type) { + case MLX5_FPGA_ACCESS_TYPE_RDMA: + ret = mem_transaction(fdev, size, addr, buf, MLX5_FPGA_WRITE); + if (ret) + return ret; + break; + case MLX5_FPGA_ACCESS_TYPE_I2C: + ret = mlx5_fpga_mem_write_i2c(fdev, size, addr, buf); + if (ret) + return ret; + break; + default: + mlx5_fpga_warn(fdev, "Unexpected write access_type %u\n", + access_type); + return -EACCES; + } + + return size; +} +EXPORT_SYMBOL(mlx5_fpga_mem_write); + +int mlx5_fpga_get_sbu_caps(struct mlx5_fpga_device *fdev, int size, void *buf) +{ + return mlx5_fpga_sbu_caps(fdev->mdev, buf, size); +} +EXPORT_SYMBOL(mlx5_fpga_get_sbu_caps); + +u64 mlx5_fpga_ddr_size_get(struct mlx5_fpga_device *fdev) +{ + return (u64)MLX5_CAP_FPGA(fdev->mdev, fpga_ddr_size) << 10; +} +EXPORT_SYMBOL(mlx5_fpga_ddr_size_get); + +u64 mlx5_fpga_ddr_base_get(struct mlx5_fpga_device *fdev) +{ + return MLX5_CAP64_FPGA(fdev->mdev, fpga_ddr_start_addr); +} +EXPORT_SYMBOL(mlx5_fpga_ddr_base_get); + +void mlx5_fpga_client_data_set(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client *client, void *data) +{ + struct mlx5_fpga_client_data *context; + + list_for_each_entry(context, &fdev->client_data_list, list) { + if (context->client != client) + continue; + context->data = data; + return; + } + + mlx5_fpga_warn(fdev, "No client context found for %s\n", client->name); +} +EXPORT_SYMBOL(mlx5_fpga_client_data_set); + +void *mlx5_fpga_client_data_get(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client *client) +{ + struct mlx5_fpga_client_data *context; + void *ret = NULL; + + list_for_each_entry(context, &fdev->client_data_list, list) { + if (context->client != client) + continue; + ret = context->data; + goto out; + } + mlx5_fpga_warn(fdev, "No client context found for %s\n", client->name); + +out: + return ret; +} +EXPORT_SYMBOL(mlx5_fpga_client_data_get); + +void mlx5_fpga_device_query(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_query *query) +{ + unsigned long flags; + + spin_lock_irqsave(&fdev->state_lock, flags); + query->image_status = fdev->image_status; + query->admin_image = fdev->last_admin_image; + query->oper_image = fdev->last_oper_image; + spin_unlock_irqrestore(&fdev->state_lock, flags); +} +EXPORT_SYMBOL(mlx5_fpga_device_query); + +int mlx5_fpga_device_reload(struct mlx5_fpga_device *fdev, + enum mlx5_fpga_image image) +{ + struct mlx5_core_dev *mdev = fdev->mdev; + unsigned long timeout; + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&fdev->state_lock, flags); + switch (fdev->fdev_state) { + case MLX5_FDEV_STATE_NONE: + err = -ENODEV; + break; + case MLX5_FDEV_STATE_IN_PROGRESS: + err = -EBUSY; + break; + case MLX5_FDEV_STATE_SUCCESS: + case MLX5_FDEV_STATE_FAILURE: + break; + } + spin_unlock_irqrestore(&fdev->state_lock, flags); + if (err) + return err; + + mutex_lock(&mdev->intf_state_mutex); + clear_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state); + + mlx5_unregister_device(mdev); + /* XXXKIB mlx5_accel_ipsec_cleanup(mdev); */ + mlx5_fpga_device_stop(mdev); + + fdev->fdev_state = MLX5_FDEV_STATE_IN_PROGRESS; + reinit_completion(&fdev->load_event); + + if (image <= MLX5_FPGA_IMAGE_MAX) { + mlx5_fpga_info(fdev, "Loading from flash\n"); + err = mlx5_fpga_load(mdev, image); + if (err) { + mlx5_fpga_err(fdev, "Failed to request load: %d\n", + err); + goto out; + } + } else { + mlx5_fpga_info(fdev, "Resetting\n"); + err = mlx5_fpga_ctrl_op(mdev, MLX5_FPGA_CTRL_OPERATION_RESET); + if (err) { + mlx5_fpga_err(fdev, "Failed to request reset: %d\n", + err); + goto out; + } + } + + timeout = jiffies + msecs_to_jiffies(MLX5_FPGA_LOAD_TIMEOUT); + err = wait_for_completion_timeout(&fdev->load_event, timeout - jiffies); + if (err < 0) { + mlx5_fpga_err(fdev, "Failed waiting for FPGA load: %d\n", err); + fdev->fdev_state = MLX5_FDEV_STATE_FAILURE; + goto out; + } + + err = mlx5_fpga_device_start(mdev); + if (err) { + mlx5_core_err(mdev, "fpga device start failed %d\n", err); + goto out; + } + /* XXXKIB err = mlx5_accel_ipsec_init(mdev); */ + if (err) { + mlx5_core_err(mdev, "IPSec device start failed %d\n", err); + goto err_fpga; + } + + err = mlx5_register_device(mdev); + if (err) { + mlx5_core_err(mdev, "mlx5_register_device failed %d\n", err); + fdev->fdev_state = MLX5_FDEV_STATE_FAILURE; + goto err_ipsec; + } + + set_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state); + goto out; + +err_ipsec: + /* XXXKIB mlx5_accel_ipsec_cleanup(mdev); */ +err_fpga: + mlx5_fpga_device_stop(mdev); +out: + mutex_unlock(&mdev->intf_state_mutex); + return err; +} +EXPORT_SYMBOL(mlx5_fpga_device_reload); + +int mlx5_fpga_flash_select(struct mlx5_fpga_device *fdev, + enum mlx5_fpga_image image) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&fdev->state_lock, flags); + switch (fdev->fdev_state) { + case MLX5_FDEV_STATE_NONE: + spin_unlock_irqrestore(&fdev->state_lock, flags); + return -ENODEV; + case MLX5_FDEV_STATE_IN_PROGRESS: + case MLX5_FDEV_STATE_SUCCESS: + case MLX5_FDEV_STATE_FAILURE: + break; + } + spin_unlock_irqrestore(&fdev->state_lock, flags); + + err = mlx5_fpga_image_select(fdev->mdev, image); + if (err) + mlx5_fpga_err(fdev, "Failed to select flash image: %d\n", err); + else + fdev->last_admin_image = image; + return err; +} +EXPORT_SYMBOL(mlx5_fpga_flash_select); + +struct device *mlx5_fpga_dev(struct mlx5_fpga_device *fdev) +{ + return &fdev->mdev->pdev->dev; +} +EXPORT_SYMBOL(mlx5_fpga_dev); + +void mlx5_fpga_get_cap(struct mlx5_fpga_device *fdev, u32 *fpga_caps) +{ + unsigned long flags; + + spin_lock_irqsave(&fdev->state_lock, flags); + memcpy(fpga_caps, &fdev->mdev->caps.fpga, sizeof(fdev->mdev->caps.fpga)); + spin_unlock_irqrestore(&fdev->state_lock, flags); +} +EXPORT_SYMBOL(mlx5_fpga_get_cap); diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_trans.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_trans.c new file mode 100644 index 000000000000..801388f37a8d --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_trans.c @@ -0,0 +1,335 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include + +enum mlx5_fpga_transaction_state { + TRANS_STATE_NONE, + TRANS_STATE_SEND, + TRANS_STATE_WAIT, + TRANS_STATE_COMPLETE, +}; + +struct mlx5_fpga_trans_priv { + const struct mlx5_fpga_transaction *user_trans; + u8 tid; + enum mlx5_fpga_transaction_state state; + u8 status; + u32 header[MLX5_ST_SZ_DW(fpga_shell_qp_packet)]; + struct mlx5_fpga_dma_buf buf; + struct list_head list_item; +}; + +struct mlx5_fpga_trans_device_state { + spinlock_t lock; /* Protects all members of this struct */ + struct list_head free_queue; + struct mlx5_fpga_trans_priv transactions[MLX5_FPGA_TID_COUNT]; +}; + +static struct mlx5_fpga_trans_priv *find_tid(struct mlx5_fpga_device *fdev, + u8 tid) +{ + if (tid >= MLX5_FPGA_TID_COUNT) { + mlx5_fpga_warn(fdev, "Unexpected transaction ID %u\n", tid); + return NULL; + } + return &fdev->trans->transactions[tid]; +} + +static struct mlx5_fpga_trans_priv *alloc_tid(struct mlx5_fpga_device *fdev) +{ + struct mlx5_fpga_trans_priv *ret; + unsigned long flags; + + spin_lock_irqsave(&fdev->trans->lock, flags); + + if (list_empty(&fdev->trans->free_queue)) { + mlx5_fpga_dbg(fdev, "No free transaction ID available\n"); + ret = NULL; + goto out; + } + + ret = list_first_entry(&fdev->trans->free_queue, + struct mlx5_fpga_trans_priv, list_item); + list_del(&ret->list_item); + + ret->state = TRANS_STATE_NONE; +out: + spin_unlock_irqrestore(&fdev->trans->lock, flags); + return ret; +} + +static void free_tid(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_trans_priv *trans_priv) +{ + unsigned long flags; + + spin_lock_irqsave(&fdev->trans->lock, flags); + list_add_tail(&trans_priv->list_item, &fdev->trans->free_queue); + spin_unlock_irqrestore(&fdev->trans->lock, flags); +} + +static void trans_complete(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_trans_priv *trans_priv, u8 status) +{ + const struct mlx5_fpga_transaction *user_trans; + unsigned long flags; + + mlx5_fpga_dbg(fdev, "Transaction %u is complete with status %u\n", + trans_priv->tid, status); + + spin_lock_irqsave(&fdev->trans->lock, flags); + trans_priv->state = TRANS_STATE_COMPLETE; + trans_priv->status = status; + spin_unlock_irqrestore(&fdev->trans->lock, flags); + + user_trans = trans_priv->user_trans; + free_tid(fdev, trans_priv); + + if (user_trans->complete1) + user_trans->complete1(user_trans, status); +} + +static void trans_send_complete(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_device *fdev, + struct mlx5_fpga_dma_buf *buf, u8 status) +{ + unsigned long flags; + struct mlx5_fpga_trans_priv *trans_priv; + + trans_priv = container_of(buf, struct mlx5_fpga_trans_priv, buf); + mlx5_fpga_dbg(fdev, "send complete tid %u. Status: %u\n", + trans_priv->tid, status); + if (status) { + trans_complete(fdev, trans_priv, status); + return; + } + + spin_lock_irqsave(&fdev->trans->lock, flags); + if (trans_priv->state == TRANS_STATE_SEND) + trans_priv->state = TRANS_STATE_WAIT; + spin_unlock_irqrestore(&fdev->trans->lock, flags); +} + +static int trans_validate(struct mlx5_fpga_device *fdev, u64 addr, size_t size) +{ + if (size > MLX5_FPGA_TRANSACTION_MAX_SIZE) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes at once. Max is %u\n", + size, MLX5_FPGA_TRANSACTION_MAX_SIZE); + return -EINVAL; + } + if (size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes. Must be full dwords\n", + size); + return -EINVAL; + } + if (size < 1) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes. Empty transaction not allowed\n", + size); + return -EINVAL; + } + if (addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes at unaligned address %jx\n", + size, (uintmax_t)addr); + return -EINVAL; + } + if ((addr >> MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS) != + ((addr + size - 1) >> MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS)) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes at address %jx. Crosses page boundary\n", + size, (uintmax_t)addr); + return -EINVAL; + } + if (addr < mlx5_fpga_ddr_base_get(fdev)) { + if (size != sizeof(u32)) { + mlx5_fpga_warn(fdev, "Cannot access %zu bytes at cr-space address %jx. Must access a single dword\n", + size, (uintmax_t)addr); + return -EINVAL; + } + } + return 0; +} + +int mlx5_fpga_trans_exec(const struct mlx5_fpga_transaction *trans) +{ + struct mlx5_fpga_conn *conn = trans->conn; + struct mlx5_fpga_trans_priv *trans_priv; + u32 *header; + int err; + + if (!trans->complete1) { + mlx5_fpga_warn(conn->fdev, "Transaction must have a completion callback\n"); + err = -EINVAL; + goto out; + } + + err = trans_validate(conn->fdev, trans->addr, trans->size); + if (err) + goto out; + + trans_priv = alloc_tid(conn->fdev); + if (!trans_priv) { + err = -EBUSY; + goto out; + } + trans_priv->user_trans = trans; + header = trans_priv->header; + + memset(header, 0, sizeof(trans_priv->header)); + memset(&trans_priv->buf, 0, sizeof(trans_priv->buf)); + MLX5_SET(fpga_shell_qp_packet, header, type, + (trans->direction == MLX5_FPGA_WRITE) ? + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_WRITE : + MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_READ); + MLX5_SET(fpga_shell_qp_packet, header, tid, trans_priv->tid); + MLX5_SET(fpga_shell_qp_packet, header, len, trans->size); + MLX5_SET64(fpga_shell_qp_packet, header, address, trans->addr); + + trans_priv->buf.sg[0].data = header; + trans_priv->buf.sg[0].size = sizeof(trans_priv->header); + if (trans->direction == MLX5_FPGA_WRITE) { + trans_priv->buf.sg[1].data = trans->data; + trans_priv->buf.sg[1].size = trans->size; + } + + trans_priv->buf.complete = trans_send_complete; + trans_priv->state = TRANS_STATE_SEND; + +#ifdef NOT_YET + /* XXXKIB */ + err = mlx5_fpga_conn_send(conn->fdev->shell_conn, &trans_priv->buf); +#else + err = 0; +#endif + if (err) + goto out_buf_tid; + goto out; + +out_buf_tid: + free_tid(conn->fdev, trans_priv); +out: + return err; +} + +void mlx5_fpga_trans_recv(void *cb_arg, struct mlx5_fpga_dma_buf *buf) +{ + struct mlx5_fpga_device *fdev = cb_arg; + struct mlx5_fpga_trans_priv *trans_priv; + size_t payload_len; + u8 status = 0; + u8 tid, type; + + mlx5_fpga_dbg(fdev, "Rx QP message on core conn; %u bytes\n", + buf->sg[0].size); + + if (buf->sg[0].size < MLX5_ST_SZ_BYTES(fpga_shell_qp_packet)) { + mlx5_fpga_warn(fdev, "Short message %u bytes from device\n", + buf->sg[0].size); + goto out; + } + payload_len = buf->sg[0].size - MLX5_ST_SZ_BYTES(fpga_shell_qp_packet); + + tid = MLX5_GET(fpga_shell_qp_packet, buf->sg[0].data, tid); + trans_priv = find_tid(fdev, tid); + if (!trans_priv) + goto out; + + type = MLX5_GET(fpga_shell_qp_packet, buf->sg[0].data, type); + switch (type) { + case MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_READ_RESPONSE: + if (trans_priv->user_trans->direction != MLX5_FPGA_READ) { + mlx5_fpga_warn(fdev, "Wrong answer type %u to a %u transaction\n", + type, trans_priv->user_trans->direction); + status = -EIO; + goto complete; + } + if (payload_len != trans_priv->user_trans->size) { + mlx5_fpga_warn(fdev, "Incorrect transaction payload length %zu expected %zu\n", + payload_len, + trans_priv->user_trans->size); + goto complete; + } + memcpy(trans_priv->user_trans->data, + MLX5_ADDR_OF(fpga_shell_qp_packet, buf->sg[0].data, + data), payload_len); + break; + case MLX5_FPGA_SHELL_QP_PACKET_TYPE_DDR_WRITE_RESPONSE: + if (trans_priv->user_trans->direction != MLX5_FPGA_WRITE) { + mlx5_fpga_warn(fdev, "Wrong answer type %u to a %u transaction\n", + type, trans_priv->user_trans->direction); + status = -EIO; + goto complete; + } + break; + default: + mlx5_fpga_warn(fdev, "Unexpected message type %u len %u from device\n", + type, buf->sg[0].size); + status = -EIO; + goto complete; + } + +complete: + trans_complete(fdev, trans_priv, status); +out: + return; +} + +int mlx5_fpga_trans_device_init(struct mlx5_fpga_device *fdev) +{ + int ret = 0; + int tid; + + fdev->trans = kzalloc(sizeof(*fdev->trans), GFP_KERNEL); + if (!fdev->trans) { + ret = -ENOMEM; + goto out; + } + + INIT_LIST_HEAD(&fdev->trans->free_queue); + for (tid = 0; tid < ARRAY_SIZE(fdev->trans->transactions); tid++) { + fdev->trans->transactions[tid].tid = tid; + list_add_tail(&fdev->trans->transactions[tid].list_item, + &fdev->trans->free_queue); + } + + spin_lock_init(&fdev->trans->lock); + +out: + return ret; +} + +void mlx5_fpga_trans_device_cleanup(struct mlx5_fpga_device *fdev) +{ + kfree(fdev->trans); +} diff --git a/sys/dev/mlx5/mlx5_fpga/mlx5fpga_xfer.c b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_xfer.c new file mode 100644 index 000000000000..85f84332dbc5 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/mlx5fpga_xfer.c @@ -0,0 +1,244 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include + +struct xfer_state { + const struct mlx5_fpga_transaction *xfer; + /* Total transactions */ + unsigned int start_count; + unsigned int done_count; + unsigned int error_count; + u8 status; + /* Inflight transactions */ + unsigned int budget; + unsigned int inflight_count; + /* Chunking state */ + size_t pos; + spinlock_t lock; /* Protects all members of this struct */ +}; + +struct xfer_transaction { + struct xfer_state *xfer_state; + struct mlx5_fpga_transaction transaction; +}; + +static void trans_complete(const struct mlx5_fpga_transaction *complete, + u8 status); + +static void xfer_complete(struct xfer_state *xfer_state) +{ + const struct mlx5_fpga_transaction *xfer = xfer_state->xfer; + u8 status = xfer_state->status; + + kfree(xfer_state); + xfer->complete1(xfer, status); +} + +/* Xfer state spin lock must be locked */ +static int exec_more(struct xfer_state *xfer_state) +{ + struct xfer_transaction *xfer_trans; + size_t left, cur_size, page_size; + u64 pos_addr, ddr_base; + u8 *pos_data; + int ret = 0; + + ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev); + page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ? + sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS); + + do { + if (xfer_state->status != IB_WC_SUCCESS) { + ret = -EIO; + break; + } + + left = xfer_state->xfer->size - xfer_state->pos; + if (!left) + break; + + xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC); + if (!xfer_trans) { + ret = -ENOMEM; + break; + } + + pos_addr = xfer_state->xfer->addr + xfer_state->pos; + pos_data = xfer_state->xfer->data + xfer_state->pos; + + /* Determine largest possible transaction at this point */ + cur_size = page_size - (pos_addr & (page_size - 1)); + if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE) + cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE; + if (cur_size > left) + cur_size = left; + + xfer_trans->xfer_state = xfer_state; + xfer_trans->transaction.addr = pos_addr; + xfer_trans->transaction.complete1 = trans_complete; + xfer_trans->transaction.conn = xfer_state->xfer->conn; + xfer_trans->transaction.data = pos_data; + xfer_trans->transaction.direction = xfer_state->xfer->direction; + xfer_trans->transaction.size = cur_size; + + xfer_state->start_count++; + xfer_state->inflight_count++; + mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n", + xfer_trans->transaction.size, + xfer_trans->transaction.data, + xfer_state->start_count, + xfer_state->inflight_count, + xfer_state->done_count, + xfer_state->error_count); + ret = mlx5_fpga_trans_exec(&xfer_trans->transaction); + if (ret) { + xfer_state->start_count--; + xfer_state->inflight_count--; + if (ret == -EBUSY) + ret = 0; + + if (ret) { + mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n", + ret, xfer_state->start_count, + xfer_state->done_count, + xfer_state->error_count); + xfer_state->status = IB_WC_GENERAL_ERR; + } + kfree(xfer_trans); + break; + } + xfer_state->pos += cur_size; + if (xfer_state->inflight_count >= xfer_state->budget) + break; + } while (cur_size != left); + + return ret; +} + +static void trans_complete(const struct mlx5_fpga_transaction *complete, + u8 status) +{ + struct xfer_transaction *xfer_trans; + struct xfer_state *xfer_state; + unsigned long flags; + bool done = false; + int ret; + + xfer_trans = container_of(complete, struct xfer_transaction, + transaction); + xfer_state = xfer_trans->xfer_state; + mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n", + xfer_trans->transaction.size, + xfer_trans->transaction.data, status, + xfer_state->start_count, xfer_state->inflight_count, + xfer_state->done_count, xfer_state->error_count); + kfree(xfer_trans); + + spin_lock_irqsave(&xfer_state->lock, flags); + + if (status != IB_WC_SUCCESS) { + xfer_state->error_count++; + mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n", + xfer_state->start_count, + xfer_state->inflight_count, + xfer_state->done_count, xfer_state->error_count); + if (xfer_state->status == IB_WC_SUCCESS) + xfer_state->status = status; + } else { + xfer_state->done_count++; + } + ret = exec_more(xfer_state); + + xfer_state->inflight_count--; + if (!xfer_state->inflight_count) + done = true; + + spin_unlock_irqrestore(&xfer_state->lock, flags); + + if (done) + xfer_complete(xfer_state); +} + +int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer) +{ + u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev); + u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev); + struct xfer_state *xfer_state; + unsigned long flags; + bool done = false; + int ret = 0; + + if (xfer->addr + xfer->size > base + size) { + mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n", + (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size)); + return -EINVAL; + } + + if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { + mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n", + (uintmax_t)xfer->addr); + return -EINVAL; + } + + if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { + mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n", + xfer->size); + return -EINVAL; + } + + if (xfer->size < 1) { + mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n", + xfer->size); + return -EINVAL; + } + + xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL); + xfer_state->xfer = xfer; + xfer_state->status = IB_WC_SUCCESS; + xfer_state->budget = 7; + spin_lock_init(&xfer_state->lock); + spin_lock_irqsave(&xfer_state->lock, flags); + + ret = exec_more(xfer_state); + if (ret && (xfer_state->start_count == 0)) + done = true; + + spin_unlock_irqrestore(&xfer_state->lock, flags); + + if (done) + xfer_complete(xfer_state); + return ret; +} diff --git a/sys/dev/mlx5/mlx5_fpga/sdk.h b/sys/dev/mlx5/mlx5_fpga/sdk.h new file mode 100644 index 000000000000..812583319850 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/sdk.h @@ -0,0 +1,368 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef MLX5_FPGA_SDK_H +#define MLX5_FPGA_SDK_H + +#include +#include +#include +/* #include */ + +#include +#include + +/** + * DOC: Innova SDK + * This header defines the in-kernel API for Innova FPGA client drivers. + */ + +#define MLX5_FPGA_CLIENT_NAME_MAX 64 + +struct mlx5_fpga_conn; +struct mlx5_fpga_device; + +/** + * struct mlx5_fpga_client - Describes an Innova client driver + */ +struct mlx5_fpga_client { + /** + * @create: Informs the client that an Innova device was created. + * The device is not yet operational at this stage + * This callback is optional + * @fdev: The FPGA device + */ + void (*create)(struct mlx5_fpga_device *fdev); + /** + * @add: Informs the client that a core device is ready and operational. + * @fdev: The FPGA device + * @param vid SBU Vendor ID + * @param pid SBU Product ID + * Any SBU-specific initialization should happen at this stage + * Return: 0 on success, nonzero error value otherwise + */ + int (*add)(struct mlx5_fpga_device *fdev, u32 vid, u16 pid); + /** + * @remove: Informs the client that a core device is not operational + * anymore. + * @fdev: The FPGA device + * SBU-specific cleanup should happen at this stage + * This callback is called once for every successful call to add() + */ + void (*remove)(struct mlx5_fpga_device *fdev); + /** + * @destroy: Informs the client that a core device is being destroyed. + * @fdev: The FPGA device + * The device is not operational at this stage + */ + void (*destroy)(struct mlx5_fpga_device *fdev); + /** The name of this client driver */ + char name[MLX5_FPGA_CLIENT_NAME_MAX]; + /** For use by core. A link in the list of client drivers */ + struct list_head list; +}; + +/** + * struct mlx5_fpga_dma_entry - A scatter-gather DMA entry + */ +struct mlx5_fpga_dma_entry { + /** @data: Virtual address pointer to the data */ + void *data; + /** @size: Size in bytes of the data */ + unsigned int size; + /** @dma_addr: Private member. Physical DMA-mapped address of the data */ + dma_addr_t dma_addr; +}; + +/** + * struct mlx5_fpga_dma_buf - A packet buffer + * May contain up to 2 scatter-gather data entries + */ +struct mlx5_fpga_dma_buf { + /** @dma_dir: DMA direction */ + enum dma_data_direction dma_dir; + /** @sg: Scatter-gather entries pointing to the data in memory */ + struct mlx5_fpga_dma_entry sg[2]; + /** @list: Item in SQ backlog, for TX packets */ + struct list_head list; + /** + * @complete: Completion routine, for TX packets + * @conn: FPGA Connection this packet was sent to + * @fdev: FPGA device this packet was sent to + * @buf: The packet buffer + * @status: 0 if successful, or an error code otherwise + */ + void (*complete)(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_device *fdev, + struct mlx5_fpga_dma_buf *buf, u8 status); +}; + +/** + * struct mlx5_fpga_conn_attr - FPGA connection attributes + * Describes the attributes of a connection + */ +struct mlx5_fpga_conn_attr { + /** @tx_size: Size of connection TX queue, in packets */ + unsigned int tx_size; + /** @rx_size: Size of connection RX queue, in packets */ + unsigned int rx_size; + /** + * @recv_cb: Callback function which is called for received packets + * @cb_arg: The value provided in mlx5_fpga_conn_attr.cb_arg + * @buf: A buffer containing a received packet + * + * buf is guaranteed to only contain a single scatter-gather entry. + * The size of the actual packet received is specified in buf.sg[0].size + * When this callback returns, the packet buffer may be re-used for + * subsequent receives. + */ + void (*recv_cb)(void *cb_arg, struct mlx5_fpga_dma_buf *buf); + void *cb_arg; +}; + +/** + * mlx5_fpga_client_register() - Register a client driver + * @client: The properties of the client driver + * + * Should be called from a client driver's module init routine. + * Note: The core will immediately callback create() and add() for any existing + * devices in the system, as well as new ones added later on. + */ +void mlx5_fpga_client_register(struct mlx5_fpga_client *client); +/** + * mlx5_fpga_client_unregister() - Unregister a client driver + * @client: The client driver to unregister + * + * Should be called from a client driver's module exit routine. + * Note: The core will immediately callback delete() and destroy() for any + * created/added devices in the system, to clean up their state. + */ +void mlx5_fpga_client_unregister(struct mlx5_fpga_client *client); + +/** + * mlx5_fpga_device_reload() - Force the FPGA to reload its synthesis from flash + * @fdev: The FPGA device + * @image: Which flash image to load + * + * This routine attempts graceful teardown of all device resources before + * loading. This includes a callback to client driver delete(). + * Calls client driver add() once device is operational again. + * Blocks until the new synthesis is loaded, and the device is fully + * initialized. + * + * Return: 0 if successful, or a negative error value otherwise + */ +int mlx5_fpga_device_reload(struct mlx5_fpga_device *fdev, + enum mlx5_fpga_image image); + +/** + * mlx5_fpga_flash_select() - Select the current active flash + * @fdev: The FPGA device + * @image: Which flash image will be active + * + * This routine selects the active flash by programming the relevant MUX. + * Useful prior to burning a new image on flash. + * This setting is volatile and is reset upon reboot or power-cycle + * + * Return: 0 if successful, or a negative error value otherwise + */ +int mlx5_fpga_flash_select(struct mlx5_fpga_device *fdev, + enum mlx5_fpga_image image); + +/** + * mlx5_fpga_sbu_conn_create() - Initialize a new FPGA SBU connection + * @fdev: The FPGA device + * @attr: Attributes of the new connection + * + * Sets up a new FPGA SBU connection with the specified attributes. + * The receive callback function may be called for incoming messages even + * before this function returns. + * + * The caller must eventually destroy the connection by calling + * mlx5_fpga_sbu_conn_destroy. + * + * Return: A new connection, or ERR_PTR() error value otherwise. + */ +struct mlx5_fpga_conn * +mlx5_fpga_sbu_conn_create(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_conn_attr *attr); + +/** + * mlx5_fpga_sbu_conn_destroy() - Destroy an FPGA SBU connection + * @conn: The FPGA SBU connection to destroy + * + * Cleans up an FPGA SBU connection which was previously created with + * mlx5_fpga_sbu_conn_create. + */ +void mlx5_fpga_sbu_conn_destroy(struct mlx5_fpga_conn *conn); + +/** + * mlx5_fpga_sbu_conn_sendmsg() - Queue the transmission of a packet + * @fdev: An FPGA SBU connection + * @buf: The packet buffer + * + * Queues a packet for transmission over an FPGA SBU connection. + * The buffer should not be modified or freed until completion. + * Upon completion, the buf's complete() callback is invoked, indicating the + * success or error status of the transmission. + * + * Return: 0 if successful, or an error value otherwise. + */ +int mlx5_fpga_sbu_conn_sendmsg(struct mlx5_fpga_conn *conn, + struct mlx5_fpga_dma_buf *buf); + +/** + * mlx5_fpga_mem_read() - Read from FPGA memory address space + * @fdev: The FPGA device + * @size: Size of chunk to read, in bytes + * @addr: Starting address to read from, in FPGA address space + * @buf: Buffer to read into + * @access_type: Method for reading + * + * Reads from the specified address into the specified buffer. + * The address may point to configuration space or to DDR. + * Large reads may be performed internally as several non-atomic operations. + * This function may sleep, so should not be called from atomic contexts. + * + * Return: 0 if successful, or an error value otherwise. + */ +int mlx5_fpga_mem_read(struct mlx5_fpga_device *fdev, size_t size, u64 addr, + void *buf, enum mlx5_fpga_access_type access_type); + +/** + * mlx5_fpga_mem_write() - Write to FPGA memory address space + * @fdev: The FPGA device + * @size: Size of chunk to write, in bytes + * @addr: Starting address to write to, in FPGA address space + * @buf: Buffer which contains data to write + * @access_type: Method for writing + * + * Writes the specified buffer data to FPGA memory at the specified address. + * The address may point to configuration space or to DDR. + * Large writes may be performed internally as several non-atomic operations. + * This function may sleep, so should not be called from atomic contexts. + * + * Return: 0 if successful, or an error value otherwise. + */ +int mlx5_fpga_mem_write(struct mlx5_fpga_device *fdev, size_t size, u64 addr, + void *buf, enum mlx5_fpga_access_type access_type); + +/** + * mlx5_fpga_get_sbu_caps() - Read the SBU capabilities + * @fdev: The FPGA device + * @size: Size of the buffer to read into + * @buf: Buffer to read the capabilities into + * + * Reads the FPGA SBU capabilities into the specified buffer. + * The format of the capabilities buffer is SBU-dependent. + * + * Return: 0 if successful + * -EINVAL if the buffer is not large enough to contain SBU caps + * or any other error value otherwise. + */ +int mlx5_fpga_get_sbu_caps(struct mlx5_fpga_device *fdev, int size, void *buf); + +/** + * mlx5_fpga_ddr_size_get() - Retrieve the size of FPGA DDR + * @fdev: The FPGA device + * + * Return: Size of DDR avaailable for FPGA, in bytes + */ +u64 mlx5_fpga_ddr_size_get(struct mlx5_fpga_device *fdev); + +/** + * mlx5_fpga_ddr_base_get() - Retrieve the base address of FPGA DDR + * @fdev: The FPGA device + * + * Return: Base address of DDR in FPGA address space + */ +u64 mlx5_fpga_ddr_base_get(struct mlx5_fpga_device *fdev); + +/** + * mlx5_fpga_client_data_set() - Attach client-defined private value to a device + * @fdev: The FPGA device + * @client: The client driver + * @data: Opaque private value + * + * Client driver may use the private value for storing device-specific + * state and configuration information, and may retrieve it with a call to + * mlx5_fpga_client_data_get(). + */ +void mlx5_fpga_client_data_set(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client *client, + void *data); + +/** + * mlx5_fpga_client_data_get() - Retrieve client-defined private value + * @fdev: The FPGA device + * @client: The client driver + * + * Client driver may use the private value for storing device-specific + * state and configuration information by calling mlx5_fpga_client_data_set() + * + * Return: The private value + */ +void *mlx5_fpga_client_data_get(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_client *client); + +/** + * mlx5_fpga_device_query() - Query FPGA device state information + * @fdev: The FPGA device + * @query: Returns the device state + * + * Queries the device state and returns it in *query + */ +void mlx5_fpga_device_query(struct mlx5_fpga_device *fdev, + struct mlx5_fpga_query *query); + +/** + * mlx5_fpga_dev() - Retrieve FPGA device structure + * @fdev: The FPGA device + + * Return: A pointer to a struct device, which may be used with dev_* logging, + * sysfs extensions, etc. + */ +struct device *mlx5_fpga_dev(struct mlx5_fpga_device *fdev); + +/** + * mlx5_fpga_get_cap() - Returns the FPGA cap mailbox from FW without parsing. + * @fdev: The FPGA device + * @fpga_caps: Is an array with a length of according to the size of + * mlx5_ifc_fpga_cap_bits/32 + * + * Returns a copy of the FPGA caps mailbox and returns it in fpga_caps + */ +void mlx5_fpga_get_cap(struct mlx5_fpga_device *fdev, u32 *fpga_caps); + +#endif /* MLX5_FPGA_SDK_H */ diff --git a/sys/dev/mlx5/mlx5_fpga/trans.h b/sys/dev/mlx5/mlx5_fpga/trans.h new file mode 100644 index 000000000000..410ab8ae1eb7 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/trans.h @@ -0,0 +1,66 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_TRANS_H__ +#define __MLX5_FPGA_TRANS_H__ + +#include +#include + +#define MLX5_FPGA_TRANSACTION_MAX_SIZE 1008 +#define MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS 3 +#define MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS 12 +#define MLX5_FPGA_TID_COUNT 256 + +enum mlx5_fpga_direction { + MLX5_FPGA_READ, + MLX5_FPGA_WRITE, +}; + +struct mlx5_fpga_transaction { + struct mlx5_fpga_conn *conn; + enum mlx5_fpga_direction direction; + size_t size; + u64 addr; + u8 *data; + void (*complete1)(const struct mlx5_fpga_transaction *complete, + u8 status); +}; + +int mlx5_fpga_trans_device_init(struct mlx5_fpga_device *fdev); +void mlx5_fpga_trans_device_cleanup(struct mlx5_fpga_device *fdev); +int mlx5_fpga_trans_exec(const struct mlx5_fpga_transaction *trans); +void mlx5_fpga_trans_recv(void *cb_arg, struct mlx5_fpga_dma_buf *buf); + +#endif /* __MLX_FPGA_TRANS_H__ */ diff --git a/sys/dev/mlx5/mlx5_fpga/xfer.h b/sys/dev/mlx5/mlx5_fpga/xfer.h new file mode 100644 index 000000000000..a409778f7d14 --- /dev/null +++ b/sys/dev/mlx5/mlx5_fpga/xfer.h @@ -0,0 +1,42 @@ +/*- + * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __MLX5_FPGA_XFER_H__ +#define __MLX5_FPGA_XFER_H__ + +#include + +int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer); + +#endif /* __MLX5_FPGA_XFER_H__ */ diff --git a/sys/dev/mlx5/mlx5_ifc.h b/sys/dev/mlx5/mlx5_ifc.h index d03ce8eda432..083daddf600b 100644 --- a/sys/dev/mlx5/mlx5_ifc.h +++ b/sys/dev/mlx5/mlx5_ifc.h @@ -28,6 +28,8 @@ #ifndef MLX5_IFC_H #define MLX5_IFC_H +#include + enum { MLX5_EVENT_TYPE_COMP = 0x0, MLX5_EVENT_TYPE_PATH_MIG = 0x1, @@ -58,7 +60,9 @@ enum { MLX5_EVENT_TYPE_DROPPED_PACKET_LOGGED_EVENT = 0x1f, MLX5_EVENT_TYPE_CMD = 0xa, MLX5_EVENT_TYPE_PAGE_REQUEST = 0xb, - MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0xd + MLX5_EVENT_TYPE_NIC_VPORT_CHANGE = 0xd, + MLX5_EVENT_TYPE_FPGA_ERROR = 0x20, + MLX5_EVENT_TYPE_FPGA_QP_ERROR = 0x21, }; enum { @@ -242,6 +246,11 @@ enum { MLX5_CMD_OP_MODIFY_FLOW_TABLE = 0x93c, MLX5_CMD_OP_ALLOC_ENCAP_HEADER = 0x93d, MLX5_CMD_OP_DEALLOC_ENCAP_HEADER = 0x93e, + MLX5_CMD_OP_FPGA_CREATE_QP = 0x960, + MLX5_CMD_OP_FPGA_MODIFY_QP = 0x961, + MLX5_CMD_OP_FPGA_QUERY_QP = 0x962, + MLX5_CMD_OP_FPGA_DESTROY_QP = 0x963, + MLX5_CMD_OP_FPGA_QUERY_QP_COUNTERS = 0x964, }; enum { @@ -998,7 +1007,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 max_tc[0x4]; u8 temp_warn_event[0x1]; u8 dcbx[0x1]; - u8 reserved_22[0x4]; + u8 general_notification_event[0x1]; + u8 reserved_at_1d3[0x2]; + u8 fpga[0x1]; u8 rol_s[0x1]; u8 rol_g[0x1]; u8 reserved_23[0x1]; diff --git a/sys/dev/mlx5/mlx5_lib/mlx5.h b/sys/dev/mlx5/mlx5_lib/mlx5.h new file mode 100644 index 000000000000..f694c24bd5b7 --- /dev/null +++ b/sys/dev/mlx5/mlx5_lib/mlx5.h @@ -0,0 +1,45 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies, Ltd. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#ifndef __LIB_MLX5_H__ +#define __LIB_MLX5_H__ + +void mlx5_init_reserved_gids(struct mlx5_core_dev *dev); +void mlx5_cleanup_reserved_gids(struct mlx5_core_dev *dev); +int mlx5_core_reserve_gids(struct mlx5_core_dev *dev, unsigned int count); +void mlx5_core_unreserve_gids(struct mlx5_core_dev *dev, unsigned int count); +int mlx5_core_reserved_gid_alloc(struct mlx5_core_dev *dev, int *gid_index); +void mlx5_core_reserved_gid_free(struct mlx5_core_dev *dev, int gid_index); + +#endif diff --git a/sys/dev/mlx5/mlx5_lib/mlx5_gid.c b/sys/dev/mlx5/mlx5_lib/mlx5_gid.c new file mode 100644 index 000000000000..5cfd0af9aff6 --- /dev/null +++ b/sys/dev/mlx5/mlx5_lib/mlx5_gid.c @@ -0,0 +1,156 @@ +/*- + * Copyright (c) 2017, Mellanox Technologies. All rights reserved. + * + * This software is available to you under a choice of one of two + * licenses. You may choose to be licensed under the terms of the GNU + * General Public License (GPL) Version 2, available from the file + * COPYING in the main directory of this source tree, or the + * OpenIB.org BSD license below: + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * - Redistributions of source code must retain the above + * copyright notice, this list of conditions and the following + * disclaimer. + * + * - Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + * $FreeBSD$ + */ + +#include +#include +#include +#include + +void mlx5_init_reserved_gids(struct mlx5_core_dev *dev) +{ + unsigned int tblsz = MLX5_CAP_ROCE(dev, roce_address_table_size); + + ida_init(&dev->roce.reserved_gids.ida); + dev->roce.reserved_gids.start = tblsz; + dev->roce.reserved_gids.count = 0; +} + +void mlx5_cleanup_reserved_gids(struct mlx5_core_dev *dev) +{ + WARN_ON(!ida_is_empty(&dev->roce.reserved_gids.ida)); + dev->roce.reserved_gids.start = 0; + dev->roce.reserved_gids.count = 0; + ida_destroy(&dev->roce.reserved_gids.ida); +} + +int mlx5_core_reserve_gids(struct mlx5_core_dev *dev, unsigned int count) +{ + if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { + mlx5_core_err(dev, "Cannot reserve GIDs when interfaces are up\n"); + return -EPERM; + } + if (dev->roce.reserved_gids.start < count) { + mlx5_core_warn(dev, "GID table exhausted attempting to reserve %d more GIDs\n", + count); + return -ENOMEM; + } + if (dev->roce.reserved_gids.count + count > MLX5_MAX_RESERVED_GIDS) { + mlx5_core_warn(dev, "Unable to reserve %d more GIDs\n", count); + return -ENOMEM; + } + + dev->roce.reserved_gids.start -= count; + dev->roce.reserved_gids.count += count; + mlx5_core_dbg(dev, "Reserved %u GIDs starting at %u\n", + dev->roce.reserved_gids.count, + dev->roce.reserved_gids.start); + return 0; +} + +void mlx5_core_unreserve_gids(struct mlx5_core_dev *dev, unsigned int count) +{ + WARN(test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state), "Unreserving GIDs when interfaces are up"); + WARN(count > dev->roce.reserved_gids.count, "Unreserving %u GIDs when only %u reserved", + count, dev->roce.reserved_gids.count); + + dev->roce.reserved_gids.start += count; + dev->roce.reserved_gids.count -= count; + mlx5_core_dbg(dev, "%u GIDs starting at %u left reserved\n", + dev->roce.reserved_gids.count, + dev->roce.reserved_gids.start); +} + +int mlx5_core_reserved_gid_alloc(struct mlx5_core_dev *dev, int *gid_index) +{ + int end = dev->roce.reserved_gids.start + + dev->roce.reserved_gids.count; + int index = 0; + + index = ida_simple_get(&dev->roce.reserved_gids.ida, + dev->roce.reserved_gids.start, end, + GFP_KERNEL); + if (index < 0) + return index; + + mlx5_core_dbg(dev, "Allocating reserved GID %u\n", index); + *gid_index = index; + return 0; +} + +void mlx5_core_reserved_gid_free(struct mlx5_core_dev *dev, int gid_index) +{ + mlx5_core_dbg(dev, "Freeing reserved GID %u\n", gid_index); + ida_simple_remove(&dev->roce.reserved_gids.ida, gid_index); +} + +unsigned int mlx5_core_reserved_gids_count(struct mlx5_core_dev *dev) +{ + return dev->roce.reserved_gids.count; +} +EXPORT_SYMBOL_GPL(mlx5_core_reserved_gids_count); + +int mlx5_core_roce_gid_set(struct mlx5_core_dev *dev, unsigned int index, + u8 roce_version, u8 roce_l3_type, const u8 *gid, + const u8 *mac, bool vlan, u16 vlan_id) +{ +#define MLX5_SET_RA(p, f, v) MLX5_SET(roce_addr_layout, p, f, v) + u32 in[MLX5_ST_SZ_DW(set_roce_address_in)] = {0}; + u32 out[MLX5_ST_SZ_DW(set_roce_address_out)] = {0}; + void *in_addr = MLX5_ADDR_OF(set_roce_address_in, in, roce_address); + char *addr_l3_addr = MLX5_ADDR_OF(roce_addr_layout, in_addr, + source_l3_address); + void *addr_mac = MLX5_ADDR_OF(roce_addr_layout, in_addr, + source_mac_47_32); + int gidsz = MLX5_FLD_SZ_BYTES(roce_addr_layout, source_l3_address); + + if (MLX5_CAP_GEN(dev, port_type) != MLX5_CAP_PORT_TYPE_ETH) + return -EINVAL; + + if (gid) { + if (vlan) { + MLX5_SET_RA(in_addr, vlan_valid, 1); + MLX5_SET_RA(in_addr, vlan_id, vlan_id); + } + + ether_addr_copy(addr_mac, mac); + MLX5_SET_RA(in_addr, roce_version, roce_version); + MLX5_SET_RA(in_addr, roce_l3_type, roce_l3_type); + memcpy(addr_l3_addr, gid, gidsz); + } + + MLX5_SET(set_roce_address_in, in, roce_address_index, index); + MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS); + return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); +} +EXPORT_SYMBOL(mlx5_core_roce_gid_set); diff --git a/sys/dev/mlx5/mlx5io.h b/sys/dev/mlx5/mlx5io.h index 5504b1e06260..de10db26cba9 100644 --- a/sys/dev/mlx5/mlx5io.h +++ b/sys/dev/mlx5/mlx5io.h @@ -57,4 +57,44 @@ struct mlx5_fwdump_get { #define MLX5_DEV_PATH _PATH_DEV"mlx5ctl" #endif +enum mlx5_fpga_id { + MLX5_FPGA_NEWTON = 0, + MLX5_FPGA_EDISON = 1, + MLX5_FPGA_MORSE = 2, +}; + +enum mlx5_fpga_image { + MLX5_FPGA_IMAGE_USER = 0, + MLX5_FPGA_IMAGE_FACTORY = 1, + MLX5_FPGA_IMAGE_MAX = MLX5_FPGA_IMAGE_FACTORY, + MLX5_FPGA_IMAGE_FACTORY_FAILOVER = 2, +}; + +enum mlx5_fpga_status { + MLX5_FPGA_STATUS_SUCCESS = 0, + MLX5_FPGA_STATUS_FAILURE = 1, + MLX5_FPGA_STATUS_IN_PROGRESS = 2, + MLX5_FPGA_STATUS_DISCONNECTED = 3, +}; + +struct mlx5_fpga_query { + enum mlx5_fpga_image admin_image; + enum mlx5_fpga_image oper_image; + enum mlx5_fpga_status image_status; +}; + +/** + * enum mlx5_fpga_access_type - Enumerated the different methods possible for + * accessing the device memory address space + */ +enum mlx5_fpga_access_type { + /** Use the slow CX-FPGA I2C bus*/ + MLX5_FPGA_ACCESS_TYPE_I2C = 0x0, + /** Use the fast 'shell QP' */ + MLX5_FPGA_ACCESS_TYPE_RDMA, + /** Use the fastest available method */ + MLX5_FPGA_ACCESS_TYPE_DONTCARE, + MLX5_FPGA_ACCESS_TYPE_MAX = MLX5_FPGA_ACCESS_TYPE_DONTCARE, +}; + #endif diff --git a/sys/modules/mlx5/Makefile b/sys/modules/mlx5/Makefile index b39da364973a..f90db5ba92d7 100644 --- a/sys/modules/mlx5/Makefile +++ b/sys/modules/mlx5/Makefile @@ -1,5 +1,7 @@ # $FreeBSD$ -.PATH: ${SRCTOP}/sys/dev/mlx5/mlx5_core +.PATH: ${SRCTOP}/sys/dev/mlx5/mlx5_core \ + ${SRCTOP}/sys/dev/mlx5/mlx5_lib \ + ${SRCTOP}/sys/dev/mlx5/mlx5_fpga KMOD=mlx5 SRCS= \ @@ -29,12 +31,23 @@ mlx5_uar.c \ mlx5_vport.c \ mlx5_vsc.c \ mlx5_wq.c \ +mlx5_gid.c \ device_if.h bus_if.h vnode_if.h pci_if.h \ opt_inet.h opt_inet6.h opt_rss.h opt_ratelimit.h CFLAGS+= -I${SRCTOP}/sys/ofed/include CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include +.if defined(CONFIG_BUILD_FPGA) +SRCS+= \ + mlx5fpga_cmd.c \ + mlx5fpga_core.c \ + mlx5fpga_sdk.c \ + mlx5fpga_trans.c \ + mlx5fpga_xfer.c \ + mlx5fpga_ipsec.c +.endif + .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS} diff --git a/sys/modules/mlx5en/Makefile b/sys/modules/mlx5en/Makefile index b413e2c32602..efac33e5e3fb 100644 --- a/sys/modules/mlx5en/Makefile +++ b/sys/modules/mlx5en/Makefile @@ -21,6 +21,10 @@ CFLAGS+= -DHAVE_PER_CQ_EVENT_PACKET CFLAGS+= -DHAVE_TCP_LRO_RX .endif +.if defined(CONFIG_BUILD_FPGA) +CFLAGS+= -DCONFIG_MLX5_FPGA +.endif + CFLAGS+= -I${SRCTOP}/sys/ofed/include CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include diff --git a/sys/modules/mlx5ib/Makefile b/sys/modules/mlx5ib/Makefile index b3b823a07195..fb9c7fbc6612 100644 --- a/sys/modules/mlx5ib/Makefile +++ b/sys/modules/mlx5ib/Makefile @@ -23,6 +23,10 @@ CFLAGS+= -I${SRCTOP}/sys/ofed/include/uapi CFLAGS+= -I${SRCTOP}/sys/compat/linuxkpi/common/include CFLAGS+= -DCONFIG_INFINIBAND_USER_MEM +.if defined(CONFIG_BUILD_FPGA) +CFLAGS+= -DCONFIG_MLX5_FPGA +.endif + .include CFLAGS+= -Wno-cast-qual -Wno-pointer-arith ${GCC_MS_EXTENSIONS}