net/mlx5: prefix all functions with mlx5
This change removes the need to distinguish unlocked priv_*() functions which are therefore renamed using a mlx5_*() prefix for consistency. At the same time, all functions from mlx5 uses a pointer to the ETH device instead of the one to the PMD private data. Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com> Acked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
This commit is contained in:
parent
7b2423cd2e
commit
af4f09f282
@ -169,8 +169,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
|
||||
(void *)dev,
|
||||
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
|
||||
/* In case mlx5_dev_stop() has not been called. */
|
||||
priv_dev_interrupt_handler_uninstall(priv, dev);
|
||||
priv_dev_traffic_disable(priv, dev);
|
||||
mlx5_dev_interrupt_handler_uninstall(dev);
|
||||
mlx5_traffic_disable(dev);
|
||||
/* Prevent crashes when queues are still in use. */
|
||||
dev->rx_pkt_burst = removed_rx_burst;
|
||||
dev->tx_pkt_burst = removed_tx_burst;
|
||||
@ -178,7 +178,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
|
||||
/* XXX race condition if mlx5_rx_burst() is still running. */
|
||||
usleep(1000);
|
||||
for (i = 0; (i != priv->rxqs_n); ++i)
|
||||
mlx5_priv_rxq_release(priv, i);
|
||||
mlx5_rxq_release(dev, i);
|
||||
priv->rxqs_n = 0;
|
||||
priv->rxqs = NULL;
|
||||
}
|
||||
@ -186,7 +186,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
|
||||
/* XXX race condition if mlx5_tx_burst() is still running. */
|
||||
usleep(1000);
|
||||
for (i = 0; (i != priv->txqs_n); ++i)
|
||||
mlx5_priv_txq_release(priv, i);
|
||||
mlx5_txq_release(dev, i);
|
||||
priv->txqs_n = 0;
|
||||
priv->txqs = NULL;
|
||||
}
|
||||
@ -201,31 +201,31 @@ mlx5_dev_close(struct rte_eth_dev *dev)
|
||||
if (priv->reta_idx != NULL)
|
||||
rte_free(priv->reta_idx);
|
||||
if (priv->primary_socket)
|
||||
priv_socket_uninit(priv);
|
||||
ret = mlx5_priv_hrxq_ibv_verify(priv);
|
||||
mlx5_socket_uninit(dev);
|
||||
ret = mlx5_hrxq_ibv_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Hash Rx queue still remain", (void *)priv);
|
||||
ret = mlx5_priv_ind_table_ibv_verify(priv);
|
||||
WARN("%p: some Hash Rx queue still remain", (void *)dev);
|
||||
ret = mlx5_ind_table_ibv_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Indirection table still remain", (void *)priv);
|
||||
ret = mlx5_priv_rxq_ibv_verify(priv);
|
||||
WARN("%p: some Indirection table still remain", (void *)dev);
|
||||
ret = mlx5_rxq_ibv_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Verbs Rx queue still remain", (void *)priv);
|
||||
ret = mlx5_priv_rxq_verify(priv);
|
||||
WARN("%p: some Verbs Rx queue still remain", (void *)dev);
|
||||
ret = mlx5_rxq_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Rx Queues still remain", (void *)priv);
|
||||
ret = mlx5_priv_txq_ibv_verify(priv);
|
||||
WARN("%p: some Rx Queues still remain", (void *)dev);
|
||||
ret = mlx5_txq_ibv_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Verbs Tx queue still remain", (void *)priv);
|
||||
ret = mlx5_priv_txq_verify(priv);
|
||||
WARN("%p: some Verbs Tx queue still remain", (void *)dev);
|
||||
ret = mlx5_txq_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Tx Queues still remain", (void *)priv);
|
||||
ret = priv_flow_verify(priv);
|
||||
WARN("%p: some Tx Queues still remain", (void *)dev);
|
||||
ret = mlx5_flow_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some flows still remain", (void *)priv);
|
||||
ret = priv_mr_verify(priv);
|
||||
WARN("%p: some flows still remain", (void *)dev);
|
||||
ret = mlx5_mr_verify(dev);
|
||||
if (ret)
|
||||
WARN("%p: some Memory Region still remain", (void *)priv);
|
||||
WARN("%p: some Memory Region still remain", (void *)dev);
|
||||
memset(priv, 0, sizeof(*priv));
|
||||
}
|
||||
|
||||
@ -466,15 +466,16 @@ static void *uar_base;
|
||||
/**
|
||||
* Reserve UAR address space for primary process.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_uar_init_primary(struct priv *priv)
|
||||
mlx5_uar_init_primary(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
void *addr = (void *)0;
|
||||
int i;
|
||||
const struct rte_mem_config *mcfg;
|
||||
@ -516,15 +517,16 @@ priv_uar_init_primary(struct priv *priv)
|
||||
* Reserve UAR address space for secondary process, align with
|
||||
* primary process.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_uar_init_secondary(struct priv *priv)
|
||||
mlx5_uar_init_secondary(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
void *addr;
|
||||
int ret;
|
||||
|
||||
@ -690,7 +692,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
struct ibv_port_attr port_attr;
|
||||
struct ibv_pd *pd = NULL;
|
||||
struct priv *priv = NULL;
|
||||
struct rte_eth_dev *eth_dev;
|
||||
struct rte_eth_dev *eth_dev = NULL;
|
||||
struct ibv_device_attr_ex device_attr_ex;
|
||||
struct ether_addr mac;
|
||||
struct ibv_device_attr_ex device_attr;
|
||||
@ -721,20 +723,19 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
}
|
||||
eth_dev->device = &pci_dev->device;
|
||||
eth_dev->dev_ops = &mlx5_dev_sec_ops;
|
||||
priv = eth_dev->data->dev_private;
|
||||
err = priv_uar_init_secondary(priv);
|
||||
err = mlx5_uar_init_secondary(eth_dev);
|
||||
if (err < 0) {
|
||||
err = -err;
|
||||
goto error;
|
||||
}
|
||||
/* Receive command fd from primary process */
|
||||
err = priv_socket_connect(priv);
|
||||
err = mlx5_socket_connect(eth_dev);
|
||||
if (err < 0) {
|
||||
err = -err;
|
||||
goto error;
|
||||
}
|
||||
/* Remap UAR for Tx queues. */
|
||||
err = priv_tx_uar_remap(priv, err);
|
||||
err = mlx5_tx_uar_remap(eth_dev, err);
|
||||
if (err)
|
||||
goto error;
|
||||
/*
|
||||
@ -743,9 +744,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
* secondary process.
|
||||
*/
|
||||
eth_dev->rx_pkt_burst =
|
||||
priv_select_rx_function(priv, eth_dev);
|
||||
mlx5_select_rx_function(eth_dev);
|
||||
eth_dev->tx_pkt_burst =
|
||||
priv_select_tx_function(priv, eth_dev);
|
||||
mlx5_select_tx_function(eth_dev);
|
||||
continue;
|
||||
}
|
||||
DEBUG("using port %u (%08" PRIx32 ")", port, test);
|
||||
@ -859,11 +860,23 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
WARN("Rx CQE compression isn't supported");
|
||||
config.cqe_comp = 0;
|
||||
}
|
||||
err = priv_uar_init_primary(priv);
|
||||
eth_dev = rte_eth_dev_allocate(name);
|
||||
if (eth_dev == NULL) {
|
||||
ERROR("can not allocate rte ethdev");
|
||||
err = ENOMEM;
|
||||
goto port_error;
|
||||
}
|
||||
eth_dev->data->dev_private = priv;
|
||||
priv->dev = eth_dev;
|
||||
eth_dev->data->mac_addrs = priv->mac;
|
||||
eth_dev->device = &pci_dev->device;
|
||||
rte_eth_copy_pci_info(eth_dev, pci_dev);
|
||||
eth_dev->device->driver = &mlx5_driver.driver;
|
||||
err = mlx5_uar_init_primary(eth_dev);
|
||||
if (err)
|
||||
goto port_error;
|
||||
/* Configure the first MAC address by default. */
|
||||
if (priv_get_mac(priv, &mac.addr_bytes)) {
|
||||
if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
|
||||
ERROR("cannot get MAC address, is mlx5_en loaded?"
|
||||
" (errno: %s)", strerror(errno));
|
||||
err = ENODEV;
|
||||
@ -878,7 +891,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
{
|
||||
char ifname[IF_NAMESIZE];
|
||||
|
||||
if (priv_get_ifname(priv, &ifname) == 0)
|
||||
if (mlx5_get_ifname(eth_dev, &ifname) == 0)
|
||||
DEBUG("port %u ifname is \"%s\"",
|
||||
priv->port, ifname);
|
||||
else
|
||||
@ -886,25 +899,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
}
|
||||
#endif
|
||||
/* Get actual MTU if possible. */
|
||||
priv_get_mtu(priv, &priv->mtu);
|
||||
mlx5_get_mtu(eth_dev, &priv->mtu);
|
||||
DEBUG("port %u MTU is %u", priv->port, priv->mtu);
|
||||
eth_dev = rte_eth_dev_allocate(name);
|
||||
if (eth_dev == NULL) {
|
||||
ERROR("can not allocate rte ethdev");
|
||||
err = ENOMEM;
|
||||
goto port_error;
|
||||
}
|
||||
eth_dev->data->dev_private = priv;
|
||||
eth_dev->data->mac_addrs = priv->mac;
|
||||
eth_dev->device = &pci_dev->device;
|
||||
rte_eth_copy_pci_info(eth_dev, pci_dev);
|
||||
eth_dev->device->driver = &mlx5_driver.driver;
|
||||
/*
|
||||
* Initialize burst functions to prevent crashes before link-up.
|
||||
*/
|
||||
eth_dev->rx_pkt_burst = removed_rx_burst;
|
||||
eth_dev->tx_pkt_burst = removed_tx_burst;
|
||||
priv->dev = eth_dev;
|
||||
eth_dev->dev_ops = &mlx5_dev_ops;
|
||||
/* Register MAC address. */
|
||||
claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
|
||||
@ -919,10 +920,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
|
||||
mlx5_glue->dv_set_context_attr(ctx,
|
||||
MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
|
||||
(void *)((uintptr_t)&alctr));
|
||||
|
||||
/* Bring Ethernet device up. */
|
||||
DEBUG("forcing Ethernet interface up");
|
||||
priv_set_flags(priv, ~IFF_UP, IFF_UP);
|
||||
mlx5_set_flags(eth_dev, ~IFF_UP, IFF_UP);
|
||||
/* Store device configuration on private structure. */
|
||||
priv->config = config;
|
||||
continue;
|
||||
|
@ -163,18 +163,16 @@ int mlx5_getenv_int(const char *);
|
||||
|
||||
/* mlx5_ethdev.c */
|
||||
|
||||
struct priv *mlx5_get_priv(struct rte_eth_dev *dev);
|
||||
int mlx5_is_secondary(void);
|
||||
int priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]);
|
||||
int priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr);
|
||||
int priv_get_mtu(struct priv *priv, uint16_t *mtu);
|
||||
int priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags);
|
||||
int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
|
||||
int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
|
||||
int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
|
||||
int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
|
||||
unsigned int flags);
|
||||
int mlx5_dev_configure(struct rte_eth_dev *dev);
|
||||
void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info);
|
||||
const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev);
|
||||
int priv_link_update(struct priv *priv, int wait_to_complete);
|
||||
int priv_force_link_status_change(struct priv *priv, int status);
|
||||
int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete);
|
||||
int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status);
|
||||
int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
|
||||
int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev,
|
||||
struct rte_eth_fc_conf *fc_conf);
|
||||
@ -183,22 +181,18 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev,
|
||||
int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
|
||||
struct rte_pci_addr *pci_addr);
|
||||
void mlx5_dev_link_status_handler(void *arg);
|
||||
void mlx5_dev_interrupt_handler(void *cb_arg);
|
||||
void priv_dev_interrupt_handler_uninstall(struct priv *priv,
|
||||
struct rte_eth_dev *dev);
|
||||
void priv_dev_interrupt_handler_install(struct priv *priv,
|
||||
struct rte_eth_dev *dev);
|
||||
void mlx5_dev_interrupt_handler(void *arg);
|
||||
void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev);
|
||||
void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev);
|
||||
int mlx5_set_link_down(struct rte_eth_dev *dev);
|
||||
int mlx5_set_link_up(struct rte_eth_dev *dev);
|
||||
eth_tx_burst_t priv_select_tx_function(struct priv *priv,
|
||||
struct rte_eth_dev *dev);
|
||||
eth_rx_burst_t priv_select_rx_function(struct priv *priv,
|
||||
struct rte_eth_dev *dev);
|
||||
int mlx5_is_removed(struct rte_eth_dev *dev);
|
||||
eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
|
||||
eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_mac.c */
|
||||
|
||||
int priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]);
|
||||
int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]);
|
||||
void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
|
||||
int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
|
||||
uint32_t index, uint32_t vmdq);
|
||||
@ -210,7 +204,7 @@ int mlx5_rss_hash_update(struct rte_eth_dev *dev,
|
||||
struct rte_eth_rss_conf *rss_conf);
|
||||
int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
|
||||
struct rte_eth_rss_conf *rss_conf);
|
||||
int priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size);
|
||||
int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size);
|
||||
int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
|
||||
struct rte_eth_rss_reta_entry64 *reta_conf,
|
||||
uint16_t reta_size);
|
||||
@ -227,13 +221,13 @@ void mlx5_allmulticast_disable(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_stats.c */
|
||||
|
||||
void priv_xstats_init(struct priv *priv);
|
||||
void mlx5_xstats_init(struct rte_eth_dev *dev);
|
||||
int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
|
||||
void mlx5_stats_reset(struct rte_eth_dev *dev);
|
||||
int mlx5_xstats_get(struct rte_eth_dev *dev,
|
||||
struct rte_eth_xstat *stats, unsigned int n);
|
||||
int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
|
||||
unsigned int n);
|
||||
void mlx5_xstats_reset(struct rte_eth_dev *dev);
|
||||
int mlx5_xstats_get_names(struct rte_eth_dev *dev,
|
||||
int mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
|
||||
struct rte_eth_xstat_name *xstats_names,
|
||||
unsigned int n);
|
||||
|
||||
@ -247,9 +241,8 @@ int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask);
|
||||
|
||||
int mlx5_dev_start(struct rte_eth_dev *dev);
|
||||
void mlx5_dev_stop(struct rte_eth_dev *dev);
|
||||
int priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev);
|
||||
int priv_dev_traffic_disable(struct priv *priv, struct rte_eth_dev *dev);
|
||||
int priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev);
|
||||
int mlx5_traffic_enable(struct rte_eth_dev *dev);
|
||||
int mlx5_traffic_disable(struct rte_eth_dev *dev);
|
||||
int mlx5_traffic_restart(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_flow.c */
|
||||
@ -259,21 +252,6 @@ int mlx5_flow_validate(struct rte_eth_dev *dev,
|
||||
const struct rte_flow_item items[],
|
||||
const struct rte_flow_action actions[],
|
||||
struct rte_flow_error *error);
|
||||
void priv_flow_flush(struct priv *priv, struct mlx5_flows *list);
|
||||
int priv_flow_create_drop_queue(struct priv *priv);
|
||||
void priv_flow_stop(struct priv *priv, struct mlx5_flows *list);
|
||||
int priv_flow_start(struct priv *priv, struct mlx5_flows *list);
|
||||
int priv_flow_verify(struct priv *priv);
|
||||
int priv_flow_create_drop_queue(struct priv *priv);
|
||||
void priv_flow_delete_drop_queue(struct priv *priv);
|
||||
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
|
||||
struct rte_flow_item_eth *eth_spec,
|
||||
struct rte_flow_item_eth *eth_mask,
|
||||
struct rte_flow_item_vlan *vlan_spec,
|
||||
struct rte_flow_item_vlan *vlan_mask);
|
||||
int mlx5_ctrl_flow(struct rte_eth_dev *dev,
|
||||
struct rte_flow_item_eth *eth_spec,
|
||||
struct rte_flow_item_eth *eth_mask);
|
||||
struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
|
||||
const struct rte_flow_attr *attr,
|
||||
const struct rte_flow_item items[],
|
||||
@ -281,6 +259,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
|
||||
struct rte_flow_error *error);
|
||||
int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
|
||||
struct rte_flow_error *error);
|
||||
void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list);
|
||||
int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
|
||||
int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
|
||||
enum rte_flow_action_type action, void *data,
|
||||
@ -291,19 +270,32 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
|
||||
enum rte_filter_type filter_type,
|
||||
enum rte_filter_op filter_op,
|
||||
void *arg);
|
||||
int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list);
|
||||
void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list);
|
||||
int mlx5_flow_verify(struct rte_eth_dev *dev);
|
||||
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
|
||||
struct rte_flow_item_eth *eth_spec,
|
||||
struct rte_flow_item_eth *eth_mask,
|
||||
struct rte_flow_item_vlan *vlan_spec,
|
||||
struct rte_flow_item_vlan *vlan_mask);
|
||||
int mlx5_ctrl_flow(struct rte_eth_dev *dev,
|
||||
struct rte_flow_item_eth *eth_spec,
|
||||
struct rte_flow_item_eth *eth_mask);
|
||||
int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev);
|
||||
void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_socket.c */
|
||||
|
||||
int priv_socket_init(struct priv *priv);
|
||||
int priv_socket_uninit(struct priv *priv);
|
||||
void priv_socket_handle(struct priv *priv);
|
||||
int priv_socket_connect(struct priv *priv);
|
||||
int mlx5_socket_init(struct rte_eth_dev *priv);
|
||||
int mlx5_socket_uninit(struct rte_eth_dev *priv);
|
||||
void mlx5_socket_handle(struct rte_eth_dev *priv);
|
||||
int mlx5_socket_connect(struct rte_eth_dev *priv);
|
||||
|
||||
/* mlx5_mr.c */
|
||||
|
||||
struct mlx5_mr *priv_mr_new(struct priv *priv, struct rte_mempool *mp);
|
||||
struct mlx5_mr *priv_mr_get(struct priv *priv, struct rte_mempool *mp);
|
||||
int priv_mr_release(struct priv *priv, struct mlx5_mr *mr);
|
||||
int priv_mr_verify(struct priv *priv);
|
||||
struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp);
|
||||
struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp);
|
||||
int mlx5_mr_release(struct mlx5_mr *mr);
|
||||
int mlx5_mr_verify(struct rte_eth_dev *dev);
|
||||
|
||||
#endif /* RTE_PMD_MLX5_H_ */
|
||||
|
@ -95,8 +95,8 @@ struct ethtool_link_settings {
|
||||
/**
|
||||
* Get interface name from private structure.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] ifname
|
||||
* Interface name output buffer.
|
||||
*
|
||||
@ -104,8 +104,9 @@ struct ethtool_link_settings {
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
int
|
||||
priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE])
|
||||
mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
DIR *dir;
|
||||
struct dirent *dent;
|
||||
unsigned int dev_type = 0;
|
||||
@ -176,8 +177,8 @@ try_dev_id:
|
||||
/**
|
||||
* Perform ifreq ioctl() on associated Ethernet device.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param req
|
||||
* Request number to pass to ioctl().
|
||||
* @param[out] ifr
|
||||
@ -187,14 +188,14 @@ try_dev_id:
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
int
|
||||
priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
|
||||
mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
|
||||
{
|
||||
int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
|
||||
int ret = -1;
|
||||
|
||||
if (sock == -1)
|
||||
return ret;
|
||||
if (priv_get_ifname(priv, &ifr->ifr_name) == 0)
|
||||
if (mlx5_get_ifname(dev, &ifr->ifr_name) == 0)
|
||||
ret = ioctl(sock, req, ifr);
|
||||
close(sock);
|
||||
return ret;
|
||||
@ -203,8 +204,8 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
|
||||
/**
|
||||
* Get device MTU.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] mtu
|
||||
* MTU value output buffer.
|
||||
*
|
||||
@ -212,10 +213,10 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
int
|
||||
priv_get_mtu(struct priv *priv, uint16_t *mtu)
|
||||
mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
|
||||
{
|
||||
struct ifreq request;
|
||||
int ret = priv_ifreq(priv, SIOCGIFMTU, &request);
|
||||
int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
|
||||
|
||||
if (ret)
|
||||
return ret;
|
||||
@ -226,8 +227,8 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu)
|
||||
/**
|
||||
* Set device MTU.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param mtu
|
||||
* MTU value to set.
|
||||
*
|
||||
@ -235,18 +236,18 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu)
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_set_mtu(struct priv *priv, uint16_t mtu)
|
||||
mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
|
||||
{
|
||||
struct ifreq request = { .ifr_mtu = mtu, };
|
||||
|
||||
return priv_ifreq(priv, SIOCSIFMTU, &request);
|
||||
return mlx5_ifreq(dev, SIOCSIFMTU, &request);
|
||||
}
|
||||
|
||||
/**
|
||||
* Set device flags.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param keep
|
||||
* Bitmask for flags that must remain untouched.
|
||||
* @param flags
|
||||
@ -256,16 +257,16 @@ priv_set_mtu(struct priv *priv, uint16_t mtu)
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
int
|
||||
priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags)
|
||||
mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
|
||||
{
|
||||
struct ifreq request;
|
||||
int ret = priv_ifreq(priv, SIOCGIFFLAGS, &request);
|
||||
int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
|
||||
|
||||
if (ret)
|
||||
return ret;
|
||||
request.ifr_flags &= keep;
|
||||
request.ifr_flags |= flags & ~keep;
|
||||
return priv_ifreq(priv, SIOCSIFFLAGS, &request);
|
||||
return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -288,11 +289,11 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
|
||||
unsigned int reta_idx_n;
|
||||
const uint8_t use_app_rss_key =
|
||||
!!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
|
||||
uint64_t supp_tx_offloads = mlx5_priv_get_tx_port_offloads(priv);
|
||||
uint64_t supp_tx_offloads = mlx5_get_tx_port_offloads(dev);
|
||||
uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
|
||||
uint64_t supp_rx_offloads =
|
||||
(mlx5_priv_get_rx_port_offloads(priv) |
|
||||
mlx5_priv_get_rx_queue_offloads(priv));
|
||||
(mlx5_get_rx_port_offloads() |
|
||||
mlx5_get_rx_queue_offloads(dev));
|
||||
uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
|
||||
|
||||
if ((tx_offloads & supp_tx_offloads) != tx_offloads) {
|
||||
@ -349,7 +350,7 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
|
||||
reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
|
||||
priv->config.ind_table_max_size :
|
||||
rxqs_n));
|
||||
if (priv_rss_reta_index_resize(priv, reta_idx_n))
|
||||
if (mlx5_rss_reta_index_resize(dev, reta_idx_n))
|
||||
return ENOMEM;
|
||||
/* When the number of RX queues is not a power of two, the remaining
|
||||
* table entries are padded with reused WQs and hashes are not spread
|
||||
@ -395,12 +396,11 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
|
||||
info->max_rx_queues = max;
|
||||
info->max_tx_queues = max;
|
||||
info->max_mac_addrs = RTE_DIM(priv->mac);
|
||||
info->rx_queue_offload_capa =
|
||||
mlx5_priv_get_rx_queue_offloads(priv);
|
||||
info->rx_offload_capa = (mlx5_priv_get_rx_port_offloads(priv) |
|
||||
info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
|
||||
info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
|
||||
info->rx_queue_offload_capa);
|
||||
info->tx_offload_capa = mlx5_priv_get_tx_port_offloads(priv);
|
||||
if (priv_get_ifname(priv, &ifname) == 0)
|
||||
info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
|
||||
if (mlx5_get_ifname(dev, &ifname) == 0)
|
||||
info->if_index = if_nametoindex(ifname);
|
||||
info->reta_size = priv->reta_idx_n ?
|
||||
priv->reta_idx_n : config->ind_table_max_size;
|
||||
@ -465,7 +465,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev)
|
||||
struct rte_eth_link dev_link;
|
||||
int link_speed = 0;
|
||||
|
||||
if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) {
|
||||
WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
@ -473,7 +473,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev)
|
||||
dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
|
||||
(ifr.ifr_flags & IFF_RUNNING));
|
||||
ifr.ifr_data = (void *)&edata;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
|
||||
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
|
||||
strerror(errno));
|
||||
return -1;
|
||||
@ -527,7 +527,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
|
||||
struct rte_eth_link dev_link;
|
||||
uint64_t sc;
|
||||
|
||||
if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) {
|
||||
WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
|
||||
return -1;
|
||||
}
|
||||
@ -535,7 +535,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
|
||||
dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
|
||||
(ifr.ifr_flags & IFF_RUNNING));
|
||||
ifr.ifr_data = (void *)&gcmd;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
|
||||
DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
|
||||
strerror(errno));
|
||||
return -1;
|
||||
@ -549,7 +549,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
|
||||
|
||||
*ecmd = gcmd;
|
||||
ifr.ifr_data = (void *)ecmd;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
|
||||
DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
|
||||
strerror(errno));
|
||||
return -1;
|
||||
@ -608,90 +608,50 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
|
||||
/**
|
||||
* Enable receiving and transmitting traffic.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
static void
|
||||
priv_link_start(struct priv *priv)
|
||||
mlx5_link_start(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct rte_eth_dev *dev = priv->dev;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int err;
|
||||
|
||||
dev->tx_pkt_burst = priv_select_tx_function(priv, dev);
|
||||
dev->rx_pkt_burst = priv_select_rx_function(priv, dev);
|
||||
err = priv_dev_traffic_enable(priv, dev);
|
||||
dev->tx_pkt_burst = mlx5_select_tx_function(dev);
|
||||
dev->rx_pkt_burst = mlx5_select_rx_function(dev);
|
||||
err = mlx5_traffic_enable(dev);
|
||||
if (err)
|
||||
ERROR("%p: error occurred while configuring control flows: %s",
|
||||
(void *)priv, strerror(err));
|
||||
err = priv_flow_start(priv, &priv->flows);
|
||||
(void *)dev, strerror(err));
|
||||
err = mlx5_flow_start(dev, &priv->flows);
|
||||
if (err)
|
||||
ERROR("%p: error occurred while configuring flows: %s",
|
||||
(void *)priv, strerror(err));
|
||||
(void *)dev, strerror(err));
|
||||
}
|
||||
|
||||
/**
|
||||
* Disable receiving and transmitting traffic.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
static void
|
||||
priv_link_stop(struct priv *priv)
|
||||
mlx5_link_stop(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct rte_eth_dev *dev = priv->dev;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
priv_flow_stop(priv, &priv->flows);
|
||||
priv_dev_traffic_disable(priv, dev);
|
||||
mlx5_flow_stop(dev, &priv->flows);
|
||||
mlx5_traffic_disable(dev);
|
||||
dev->rx_pkt_burst = removed_rx_burst;
|
||||
dev->tx_pkt_burst = removed_tx_burst;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieve physical link information and update rx/tx_pkt_burst callbacks
|
||||
* accordingly.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param wait_to_complete
|
||||
* Wait for request completion (ignored).
|
||||
*/
|
||||
int
|
||||
priv_link_update(struct priv *priv, int wait_to_complete __rte_unused)
|
||||
{
|
||||
struct rte_eth_dev *dev = priv->dev;
|
||||
struct utsname utsname;
|
||||
int ver[3];
|
||||
int ret;
|
||||
struct rte_eth_link dev_link = dev->data->dev_link;
|
||||
|
||||
if (uname(&utsname) == -1 ||
|
||||
sscanf(utsname.release, "%d.%d.%d",
|
||||
&ver[0], &ver[1], &ver[2]) != 3 ||
|
||||
KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0))
|
||||
ret = mlx5_link_update_unlocked_gset(dev);
|
||||
else
|
||||
ret = mlx5_link_update_unlocked_gs(dev);
|
||||
/* If lsc interrupt is disabled, should always be ready for traffic. */
|
||||
if (!dev->data->dev_conf.intr_conf.lsc) {
|
||||
priv_link_start(priv);
|
||||
return ret;
|
||||
}
|
||||
/* Re-select burst callbacks only if link status has been changed. */
|
||||
if (!ret && dev_link.link_status != dev->data->dev_link.link_status) {
|
||||
if (dev->data->dev_link.link_status == ETH_LINK_UP)
|
||||
priv_link_start(priv);
|
||||
else
|
||||
priv_link_stop(priv);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Querying the link status till it changes to the desired state.
|
||||
* Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param status
|
||||
* Link desired status.
|
||||
*
|
||||
@ -699,13 +659,13 @@ priv_link_update(struct priv *priv, int wait_to_complete __rte_unused)
|
||||
* 0 on success, negative errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_force_link_status_change(struct priv *priv, int status)
|
||||
mlx5_force_link_status_change(struct rte_eth_dev *dev, int status)
|
||||
{
|
||||
int try = 0;
|
||||
|
||||
while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) {
|
||||
priv_link_update(priv, 0);
|
||||
if (priv->dev->data->dev_link.link_status == status)
|
||||
mlx5_link_update(dev, 0);
|
||||
if (dev->data->dev_link.link_status == status)
|
||||
return 0;
|
||||
try++;
|
||||
sleep(1);
|
||||
@ -727,10 +687,30 @@ priv_force_link_status_change(struct priv *priv, int status)
|
||||
int
|
||||
mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct utsname utsname;
|
||||
int ver[3];
|
||||
int ret;
|
||||
struct rte_eth_link dev_link = dev->data->dev_link;
|
||||
|
||||
ret = priv_link_update(priv, wait_to_complete);
|
||||
if (uname(&utsname) == -1 ||
|
||||
sscanf(utsname.release, "%d.%d.%d",
|
||||
&ver[0], &ver[1], &ver[2]) != 3 ||
|
||||
KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0))
|
||||
ret = mlx5_link_update_unlocked_gset(dev);
|
||||
else
|
||||
ret = mlx5_link_update_unlocked_gs(dev);
|
||||
/* If lsc interrupt is disabled, should always be ready for traffic. */
|
||||
if (!dev->data->dev_conf.intr_conf.lsc) {
|
||||
mlx5_link_start(dev);
|
||||
return ret;
|
||||
}
|
||||
/* Re-select burst callbacks only if link status has been changed. */
|
||||
if (!ret && dev_link.link_status != dev->data->dev_link.link_status) {
|
||||
if (dev->data->dev_link.link_status == ETH_LINK_UP)
|
||||
mlx5_link_start(dev);
|
||||
else
|
||||
mlx5_link_stop(dev);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -752,14 +732,14 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
|
||||
uint16_t kern_mtu;
|
||||
int ret = 0;
|
||||
|
||||
ret = priv_get_mtu(priv, &kern_mtu);
|
||||
ret = mlx5_get_mtu(dev, &kern_mtu);
|
||||
if (ret)
|
||||
goto out;
|
||||
/* Set kernel interface MTU first. */
|
||||
ret = priv_set_mtu(priv, mtu);
|
||||
ret = mlx5_set_mtu(dev, mtu);
|
||||
if (ret)
|
||||
goto out;
|
||||
ret = priv_get_mtu(priv, &kern_mtu);
|
||||
ret = mlx5_get_mtu(dev, &kern_mtu);
|
||||
if (ret)
|
||||
goto out;
|
||||
if (kern_mtu == mtu) {
|
||||
@ -789,7 +769,6 @@ out:
|
||||
int
|
||||
mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct ifreq ifr;
|
||||
struct ethtool_pauseparam ethpause = {
|
||||
.cmd = ETHTOOL_GPAUSEPARAM
|
||||
@ -797,10 +776,9 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
|
||||
int ret;
|
||||
|
||||
ifr.ifr_data = (void *)ðpause;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
|
||||
ret = errno;
|
||||
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)"
|
||||
" failed: %s",
|
||||
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed: %s",
|
||||
strerror(ret));
|
||||
goto out;
|
||||
}
|
||||
@ -833,7 +811,6 @@ out:
|
||||
int
|
||||
mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct ifreq ifr;
|
||||
struct ethtool_pauseparam ethpause = {
|
||||
.cmd = ETHTOOL_SPAUSEPARAM
|
||||
@ -853,7 +830,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
|
||||
ethpause.tx_pause = 1;
|
||||
else
|
||||
ethpause.tx_pause = 0;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
|
||||
ret = errno;
|
||||
WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
|
||||
" failed: %s",
|
||||
@ -919,18 +896,19 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
|
||||
/**
|
||||
* Update the link status.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* Zero if the callback process can be called immediately.
|
||||
*/
|
||||
static int
|
||||
priv_link_status_update(struct priv *priv)
|
||||
mlx5_link_status_update(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct rte_eth_link *link = &priv->dev->data->dev_link;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_eth_link *link = &dev->data->dev_link;
|
||||
|
||||
priv_link_update(priv, 0);
|
||||
mlx5_link_update(dev, 0);
|
||||
if (((link->link_speed == 0) && link->link_status) ||
|
||||
((link->link_speed != 0) && !link->link_status)) {
|
||||
/*
|
||||
@ -955,8 +933,8 @@ priv_link_status_update(struct priv *priv)
|
||||
/**
|
||||
* Device status handler.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param events
|
||||
* Pointer to event flags holder.
|
||||
*
|
||||
@ -964,8 +942,9 @@ priv_link_status_update(struct priv *priv)
|
||||
* Events bitmap of callback process which can be called immediately.
|
||||
*/
|
||||
static uint32_t
|
||||
priv_dev_status_handler(struct priv *priv)
|
||||
mlx5_dev_status_handler(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct ibv_async_event event;
|
||||
uint32_t ret = 0;
|
||||
|
||||
@ -975,10 +954,10 @@ priv_dev_status_handler(struct priv *priv)
|
||||
break;
|
||||
if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
|
||||
event.event_type == IBV_EVENT_PORT_ERR) &&
|
||||
(priv->dev->data->dev_conf.intr_conf.lsc == 1))
|
||||
(dev->data->dev_conf.intr_conf.lsc == 1))
|
||||
ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
|
||||
else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
|
||||
priv->dev->data->dev_conf.intr_conf.rmv == 1)
|
||||
dev->data->dev_conf.intr_conf.rmv == 1)
|
||||
ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
|
||||
else
|
||||
DEBUG("event type %d on port %d not handled",
|
||||
@ -986,7 +965,7 @@ priv_dev_status_handler(struct priv *priv)
|
||||
mlx5_glue->ack_async_event(&event);
|
||||
}
|
||||
if (ret & (1 << RTE_ETH_EVENT_INTR_LSC))
|
||||
if (priv_link_status_update(priv))
|
||||
if (mlx5_link_status_update(dev))
|
||||
ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC);
|
||||
return ret;
|
||||
}
|
||||
@ -1005,7 +984,7 @@ mlx5_dev_link_status_handler(void *arg)
|
||||
int ret;
|
||||
|
||||
priv->pending_alarm = 0;
|
||||
ret = priv_link_status_update(priv);
|
||||
ret = mlx5_link_status_update(dev);
|
||||
if (!ret)
|
||||
_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
|
||||
}
|
||||
@ -1022,10 +1001,9 @@ void
|
||||
mlx5_dev_interrupt_handler(void *cb_arg)
|
||||
{
|
||||
struct rte_eth_dev *dev = cb_arg;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
uint32_t events;
|
||||
|
||||
events = priv_dev_status_handler(priv);
|
||||
events = mlx5_dev_status_handler(dev);
|
||||
if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
|
||||
_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
|
||||
if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
|
||||
@ -1042,22 +1020,21 @@ static void
|
||||
mlx5_dev_handler_socket(void *cb_arg)
|
||||
{
|
||||
struct rte_eth_dev *dev = cb_arg;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
priv_socket_handle(priv);
|
||||
mlx5_socket_handle(dev);
|
||||
}
|
||||
|
||||
/**
|
||||
* Uninstall interrupt handler.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to the rte_eth_dev structure.
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
|
||||
mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
if (dev->data->dev_conf.intr_conf.lsc ||
|
||||
dev->data->dev_conf.intr_conf.rmv)
|
||||
rte_intr_callback_unregister(&priv->intr_handle,
|
||||
@ -1078,14 +1055,13 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
|
||||
/**
|
||||
* Install interrupt handler.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to the rte_eth_dev structure.
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
|
||||
mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int rc, flags;
|
||||
|
||||
assert(priv->ctx->async_fd > 0);
|
||||
@ -1103,7 +1079,7 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
|
||||
rte_intr_callback_register(&priv->intr_handle,
|
||||
mlx5_dev_interrupt_handler, dev);
|
||||
}
|
||||
rc = priv_socket_init(priv);
|
||||
rc = mlx5_socket_init(dev);
|
||||
if (!rc && priv->primary_socket) {
|
||||
priv->intr_handle_socket.fd = priv->primary_socket;
|
||||
priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
|
||||
@ -1112,23 +1088,6 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Change the link state (UP / DOWN).
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private data structure.
|
||||
* @param up
|
||||
* Nonzero for link up, otherwise link down.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_dev_set_link(struct priv *priv, int up)
|
||||
{
|
||||
return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP);
|
||||
}
|
||||
|
||||
/**
|
||||
* DPDK callback to bring the link DOWN.
|
||||
*
|
||||
@ -1141,11 +1100,7 @@ priv_dev_set_link(struct priv *priv, int up)
|
||||
int
|
||||
mlx5_set_link_down(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int err;
|
||||
|
||||
err = priv_dev_set_link(priv, 0);
|
||||
return err;
|
||||
return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -1160,27 +1115,22 @@ mlx5_set_link_down(struct rte_eth_dev *dev)
|
||||
int
|
||||
mlx5_set_link_up(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int err;
|
||||
|
||||
err = priv_dev_set_link(priv, 1);
|
||||
return err;
|
||||
return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
|
||||
}
|
||||
|
||||
/**
|
||||
* Configure the TX function to use.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private data structure.
|
||||
* @param dev
|
||||
* Pointer to rte_eth_dev structure.
|
||||
* Pointer to private data structure.
|
||||
*
|
||||
* @return
|
||||
* Pointer to selected Tx burst function.
|
||||
*/
|
||||
eth_tx_burst_t
|
||||
priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
|
||||
mlx5_select_tx_function(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
|
||||
struct mlx5_dev_config *config = &priv->config;
|
||||
uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
|
||||
@ -1194,8 +1144,8 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
|
||||
if (vlan_insert || tso)
|
||||
return tx_pkt_burst;
|
||||
if (config->mps == MLX5_MPW_ENHANCED) {
|
||||
if (priv_check_vec_tx_support(priv, dev) > 0) {
|
||||
if (priv_check_raw_vec_tx_support(priv, dev) > 0)
|
||||
if (mlx5_check_vec_tx_support(dev) > 0) {
|
||||
if (mlx5_check_raw_vec_tx_support(dev) > 0)
|
||||
tx_pkt_burst = mlx5_tx_burst_raw_vec;
|
||||
else
|
||||
tx_pkt_burst = mlx5_tx_burst_vec;
|
||||
@ -1217,21 +1167,19 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
|
||||
/**
|
||||
* Configure the RX function to use.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private data structure.
|
||||
* @param dev
|
||||
* Pointer to rte_eth_dev structure.
|
||||
* Pointer to private data structure.
|
||||
*
|
||||
* @return
|
||||
* Pointer to selected Rx burst function.
|
||||
*/
|
||||
eth_rx_burst_t
|
||||
priv_select_rx_function(struct priv *priv, __rte_unused struct rte_eth_dev *dev)
|
||||
mlx5_select_rx_function(struct rte_eth_dev *dev)
|
||||
{
|
||||
eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
|
||||
|
||||
assert(priv != NULL);
|
||||
if (priv_check_vec_rx_support(priv) > 0) {
|
||||
assert(dev != NULL);
|
||||
if (mlx5_check_vec_rx_support(dev) > 0) {
|
||||
rx_pkt_burst = mlx5_rx_burst_vec;
|
||||
DEBUG("selected RX vectorized function");
|
||||
}
|
||||
|
@ -89,7 +89,7 @@ static int
|
||||
mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
|
||||
|
||||
static int
|
||||
mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
|
||||
mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
|
||||
|
||||
/* Hash RX queue types. */
|
||||
enum hash_rxq_type {
|
||||
@ -515,8 +515,6 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
|
||||
* Copy the RSS configuration from the user ones, of the rss_conf is null,
|
||||
* uses the driver one.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param parser
|
||||
* Internal parser structure.
|
||||
* @param rss_conf
|
||||
@ -526,13 +524,12 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_flow_convert_rss_conf(struct priv *priv __rte_unused,
|
||||
struct mlx5_flow_parse *parser,
|
||||
mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser,
|
||||
const struct rte_eth_rss_conf *rss_conf)
|
||||
{
|
||||
/*
|
||||
* This function is also called at the beginning of
|
||||
* priv_flow_convert_actions() to initialize the parser with the
|
||||
* mlx5_flow_convert_actions() to initialize the parser with the
|
||||
* device default RSS configuration.
|
||||
*/
|
||||
if (rss_conf) {
|
||||
@ -554,23 +551,17 @@ priv_flow_convert_rss_conf(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Extract attribute to the parser.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] attr
|
||||
* Flow rule attributes.
|
||||
* @param[out] error
|
||||
* Perform verbose error reporting if not NULL.
|
||||
* @param[in, out] parser
|
||||
* Internal parser structure.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_convert_attributes(struct priv *priv __rte_unused,
|
||||
const struct rte_flow_attr *attr,
|
||||
struct rte_flow_error *error,
|
||||
struct mlx5_flow_parse *parser __rte_unused)
|
||||
mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
if (attr->group) {
|
||||
rte_flow_error_set(error, ENOTSUP,
|
||||
@ -606,8 +597,8 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Extract actions request to the parser.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[in] actions
|
||||
* Associated actions (list terminated by the END action).
|
||||
* @param[out] error
|
||||
@ -619,16 +610,18 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused,
|
||||
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_convert_actions(struct priv *priv,
|
||||
mlx5_flow_convert_actions(struct rte_eth_dev *dev,
|
||||
const struct rte_flow_action actions[],
|
||||
struct rte_flow_error *error,
|
||||
struct mlx5_flow_parse *parser)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
/*
|
||||
* Add default RSS configuration necessary for Verbs to create QP even
|
||||
* if no RSS is necessary.
|
||||
*/
|
||||
priv_flow_convert_rss_conf(priv, parser,
|
||||
mlx5_flow_convert_rss_conf(parser,
|
||||
(const struct rte_eth_rss_conf *)
|
||||
&priv->rss_conf);
|
||||
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
|
||||
@ -708,8 +701,7 @@ priv_flow_convert_actions(struct priv *priv,
|
||||
for (n = 0; n < rss->num; ++n)
|
||||
parser->queues[n] = rss->queue[n];
|
||||
parser->queues_n = rss->num;
|
||||
if (priv_flow_convert_rss_conf(priv, parser,
|
||||
rss->rss_conf)) {
|
||||
if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) {
|
||||
rte_flow_error_set(error, EINVAL,
|
||||
RTE_FLOW_ERROR_TYPE_ACTION,
|
||||
actions,
|
||||
@ -763,8 +755,6 @@ exit_action_not_supported:
|
||||
/**
|
||||
* Validate items.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] items
|
||||
* Pattern specification (list terminated by the END pattern item).
|
||||
* @param[out] error
|
||||
@ -776,8 +766,7 @@ exit_action_not_supported:
|
||||
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_convert_items_validate(struct priv *priv __rte_unused,
|
||||
const struct rte_flow_item items[],
|
||||
mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
|
||||
struct rte_flow_error *error,
|
||||
struct mlx5_flow_parse *parser)
|
||||
{
|
||||
@ -854,8 +843,6 @@ exit_item_not_supported:
|
||||
/**
|
||||
* Allocate memory space to store verbs flow attributes.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] priority
|
||||
* Flow priority.
|
||||
* @param[in] size
|
||||
@ -867,8 +854,7 @@ exit_item_not_supported:
|
||||
* A verbs flow attribute on success, NULL otherwise.
|
||||
*/
|
||||
static struct ibv_flow_attr *
|
||||
priv_flow_convert_allocate(struct priv *priv __rte_unused,
|
||||
unsigned int priority,
|
||||
mlx5_flow_convert_allocate(unsigned int priority,
|
||||
unsigned int size,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
@ -889,14 +875,11 @@ priv_flow_convert_allocate(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Finalise verbs flow attributes.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in, out] parser
|
||||
* Internal parser structure.
|
||||
*/
|
||||
static void
|
||||
priv_flow_convert_finalise(struct priv *priv __rte_unused,
|
||||
struct mlx5_flow_parse *parser)
|
||||
mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
|
||||
{
|
||||
const unsigned int ipv4 =
|
||||
hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
|
||||
@ -1015,8 +998,8 @@ fill:
|
||||
/**
|
||||
* Validate and convert a flow supported by the NIC.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[in] attr
|
||||
* Flow rule attributes.
|
||||
* @param[in] pattern
|
||||
@ -1032,7 +1015,7 @@ fill:
|
||||
* 0 on success, a negative errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_convert(struct priv *priv,
|
||||
mlx5_flow_convert(struct rte_eth_dev *dev,
|
||||
const struct rte_flow_attr *attr,
|
||||
const struct rte_flow_item items[],
|
||||
const struct rte_flow_action actions[],
|
||||
@ -1049,16 +1032,16 @@ priv_flow_convert(struct priv *priv,
|
||||
.layer = HASH_RXQ_ETH,
|
||||
.mark_id = MLX5_FLOW_MARK_DEFAULT,
|
||||
};
|
||||
ret = priv_flow_convert_attributes(priv, attr, error, parser);
|
||||
ret = mlx5_flow_convert_attributes(attr, error);
|
||||
if (ret)
|
||||
return ret;
|
||||
ret = priv_flow_convert_actions(priv, actions, error, parser);
|
||||
ret = mlx5_flow_convert_actions(dev, actions, error, parser);
|
||||
if (ret)
|
||||
return ret;
|
||||
ret = priv_flow_convert_items_validate(priv, items, error, parser);
|
||||
ret = mlx5_flow_convert_items_validate(items, error, parser);
|
||||
if (ret)
|
||||
return ret;
|
||||
priv_flow_convert_finalise(priv, parser);
|
||||
mlx5_flow_convert_finalise(parser);
|
||||
/*
|
||||
* Second step.
|
||||
* Allocate the memory space to store verbs specifications.
|
||||
@ -1070,8 +1053,7 @@ priv_flow_convert(struct priv *priv,
|
||||
unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
|
||||
|
||||
parser->queue[HASH_RXQ_ETH].ibv_attr =
|
||||
priv_flow_convert_allocate(priv, priority,
|
||||
offset, error);
|
||||
mlx5_flow_convert_allocate(priority, offset, error);
|
||||
if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
|
||||
return ENOMEM;
|
||||
parser->queue[HASH_RXQ_ETH].offset =
|
||||
@ -1089,7 +1071,7 @@ priv_flow_convert(struct priv *priv,
|
||||
continue;
|
||||
offset = parser->queue[i].offset;
|
||||
parser->queue[i].ibv_attr =
|
||||
priv_flow_convert_allocate(priv, priority,
|
||||
mlx5_flow_convert_allocate(priority,
|
||||
offset, error);
|
||||
if (!parser->queue[i].ibv_attr)
|
||||
goto exit_enomem;
|
||||
@ -1117,7 +1099,7 @@ priv_flow_convert(struct priv *priv,
|
||||
if (parser->mark)
|
||||
mlx5_flow_create_flag_mark(parser, parser->mark_id);
|
||||
if (parser->count && parser->create) {
|
||||
mlx5_flow_create_count(priv, parser);
|
||||
mlx5_flow_create_count(dev, parser);
|
||||
if (!parser->cs)
|
||||
goto exit_count_error;
|
||||
}
|
||||
@ -1126,7 +1108,7 @@ priv_flow_convert(struct priv *priv,
|
||||
* configuration.
|
||||
*/
|
||||
if (!parser->drop) {
|
||||
priv_flow_convert_finalise(priv, parser);
|
||||
mlx5_flow_convert_finalise(parser);
|
||||
} else {
|
||||
parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
|
||||
attr->priority +
|
||||
@ -1578,8 +1560,8 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
|
||||
/**
|
||||
* Convert count action to Verbs specification.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param parser
|
||||
* Pointer to MLX5 flow parser structure.
|
||||
*
|
||||
@ -1587,10 +1569,11 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
mlx5_flow_create_count(struct priv *priv __rte_unused,
|
||||
mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
|
||||
struct mlx5_flow_parse *parser __rte_unused)
|
||||
{
|
||||
#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
|
||||
struct ibv_counter_set_init_attr init_attr = {0};
|
||||
struct ibv_flow_spec_counter_action counter = {
|
||||
@ -1612,8 +1595,8 @@ mlx5_flow_create_count(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Complete flow rule creation with a drop queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param parser
|
||||
* Internal parser structure.
|
||||
* @param flow
|
||||
@ -1625,11 +1608,12 @@ mlx5_flow_create_count(struct priv *priv __rte_unused,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_flow_create_action_queue_drop(struct priv *priv,
|
||||
mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
|
||||
struct mlx5_flow_parse *parser,
|
||||
struct rte_flow *flow,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct ibv_flow_spec_action_drop *drop;
|
||||
unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
|
||||
int err = 0;
|
||||
@ -1684,8 +1668,8 @@ error:
|
||||
/**
|
||||
* Create hash Rx queues when RSS is enabled.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param parser
|
||||
* Internal parser structure.
|
||||
* @param flow
|
||||
@ -1697,11 +1681,12 @@ error:
|
||||
* 0 on success, a errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_create_action_queue_rss(struct priv *priv,
|
||||
mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
|
||||
struct mlx5_flow_parse *parser,
|
||||
struct rte_flow *flow,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i != hash_rxq_init_n; ++i) {
|
||||
@ -1715,21 +1700,21 @@ priv_flow_create_action_queue_rss(struct priv *priv,
|
||||
if (!priv->dev->data->dev_started)
|
||||
continue;
|
||||
flow->frxq[i].hrxq =
|
||||
mlx5_priv_hrxq_get(priv,
|
||||
parser->rss_conf.rss_key,
|
||||
parser->rss_conf.rss_key_len,
|
||||
hash_fields,
|
||||
parser->queues,
|
||||
parser->queues_n);
|
||||
mlx5_hrxq_get(dev,
|
||||
parser->rss_conf.rss_key,
|
||||
parser->rss_conf.rss_key_len,
|
||||
hash_fields,
|
||||
parser->queues,
|
||||
parser->queues_n);
|
||||
if (flow->frxq[i].hrxq)
|
||||
continue;
|
||||
flow->frxq[i].hrxq =
|
||||
mlx5_priv_hrxq_new(priv,
|
||||
parser->rss_conf.rss_key,
|
||||
parser->rss_conf.rss_key_len,
|
||||
hash_fields,
|
||||
parser->queues,
|
||||
parser->queues_n);
|
||||
mlx5_hrxq_new(dev,
|
||||
parser->rss_conf.rss_key,
|
||||
parser->rss_conf.rss_key_len,
|
||||
hash_fields,
|
||||
parser->queues,
|
||||
parser->queues_n);
|
||||
if (!flow->frxq[i].hrxq) {
|
||||
rte_flow_error_set(error, ENOMEM,
|
||||
RTE_FLOW_ERROR_TYPE_HANDLE,
|
||||
@ -1743,8 +1728,8 @@ priv_flow_create_action_queue_rss(struct priv *priv,
|
||||
/**
|
||||
* Complete flow rule creation.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param parser
|
||||
* Internal parser structure.
|
||||
* @param flow
|
||||
@ -1756,11 +1741,12 @@ priv_flow_create_action_queue_rss(struct priv *priv,
|
||||
* 0 on success, a errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_create_action_queue(struct priv *priv,
|
||||
mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
|
||||
struct mlx5_flow_parse *parser,
|
||||
struct rte_flow *flow,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int err = 0;
|
||||
unsigned int i;
|
||||
unsigned int flows_n = 0;
|
||||
@ -1768,7 +1754,7 @@ priv_flow_create_action_queue(struct priv *priv,
|
||||
assert(priv->pd);
|
||||
assert(priv->ctx);
|
||||
assert(!parser->drop);
|
||||
err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
|
||||
err = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
|
||||
if (err)
|
||||
goto error;
|
||||
if (parser->count)
|
||||
@ -1815,7 +1801,7 @@ error:
|
||||
claim_zero(mlx5_glue->destroy_flow(ibv_flow));
|
||||
}
|
||||
if (flow->frxq[i].hrxq)
|
||||
mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
|
||||
mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
|
||||
if (flow->frxq[i].ibv_attr)
|
||||
rte_free(flow->frxq[i].ibv_attr);
|
||||
}
|
||||
@ -1830,8 +1816,8 @@ error:
|
||||
/**
|
||||
* Convert a flow.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param list
|
||||
* Pointer to a TAILQ flow list.
|
||||
* @param[in] attr
|
||||
@ -1847,19 +1833,19 @@ error:
|
||||
* A flow on success, NULL otherwise.
|
||||
*/
|
||||
static struct rte_flow *
|
||||
priv_flow_create(struct priv *priv,
|
||||
struct mlx5_flows *list,
|
||||
const struct rte_flow_attr *attr,
|
||||
const struct rte_flow_item items[],
|
||||
const struct rte_flow_action actions[],
|
||||
struct rte_flow_error *error)
|
||||
mlx5_flow_list_create(struct rte_eth_dev *dev,
|
||||
struct mlx5_flows *list,
|
||||
const struct rte_flow_attr *attr,
|
||||
const struct rte_flow_item items[],
|
||||
const struct rte_flow_action actions[],
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct mlx5_flow_parse parser = { .create = 1, };
|
||||
struct rte_flow *flow = NULL;
|
||||
unsigned int i;
|
||||
int err;
|
||||
|
||||
err = priv_flow_convert(priv, attr, items, actions, error, &parser);
|
||||
err = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
|
||||
if (err)
|
||||
goto exit;
|
||||
flow = rte_calloc(__func__, 1,
|
||||
@ -1883,10 +1869,10 @@ priv_flow_create(struct priv *priv,
|
||||
memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
|
||||
/* finalise the flow. */
|
||||
if (parser.drop)
|
||||
err = priv_flow_create_action_queue_drop(priv, &parser, flow,
|
||||
err = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
|
||||
error);
|
||||
else
|
||||
err = priv_flow_create_action_queue(priv, &parser, flow, error);
|
||||
err = mlx5_flow_create_action_queue(dev, &parser, flow, error);
|
||||
if (err)
|
||||
goto exit;
|
||||
TAILQ_INSERT_TAIL(list, flow, next);
|
||||
@ -1915,11 +1901,10 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
|
||||
const struct rte_flow_action actions[],
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int ret;
|
||||
struct mlx5_flow_parse parser = { .create = 0, };
|
||||
|
||||
ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
|
||||
ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -1937,28 +1922,26 @@ mlx5_flow_create(struct rte_eth_dev *dev,
|
||||
struct rte_flow_error *error)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_flow *flow;
|
||||
|
||||
flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
|
||||
error);
|
||||
return flow;
|
||||
return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
|
||||
error);
|
||||
}
|
||||
|
||||
/**
|
||||
* Destroy a flow.
|
||||
* Destroy a flow in a list.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param list
|
||||
* Pointer to a TAILQ flow list.
|
||||
* @param[in] flow
|
||||
* Flow to destroy.
|
||||
*/
|
||||
static void
|
||||
priv_flow_destroy(struct priv *priv,
|
||||
struct mlx5_flows *list,
|
||||
struct rte_flow *flow)
|
||||
mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
|
||||
struct rte_flow *flow)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
|
||||
if (flow->drop || !flow->mark)
|
||||
@ -2006,7 +1989,7 @@ free:
|
||||
claim_zero(mlx5_glue->destroy_flow
|
||||
(frxq->ibv_flow));
|
||||
if (frxq->hrxq)
|
||||
mlx5_priv_hrxq_release(priv, frxq->hrxq);
|
||||
mlx5_hrxq_release(dev, frxq->hrxq);
|
||||
if (frxq->ibv_attr)
|
||||
rte_free(frxq->ibv_attr);
|
||||
}
|
||||
@ -2023,34 +2006,35 @@ free:
|
||||
/**
|
||||
* Destroy all flows.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param list
|
||||
* Pointer to a TAILQ flow list.
|
||||
*/
|
||||
void
|
||||
priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
|
||||
mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
|
||||
{
|
||||
while (!TAILQ_EMPTY(list)) {
|
||||
struct rte_flow *flow;
|
||||
|
||||
flow = TAILQ_FIRST(list);
|
||||
priv_flow_destroy(priv, list, flow);
|
||||
mlx5_flow_list_destroy(dev, list, flow);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create drop queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success.
|
||||
*/
|
||||
int
|
||||
priv_flow_create_drop_queue(struct priv *priv)
|
||||
mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_hrxq_drop *fdq = NULL;
|
||||
|
||||
assert(priv->pd);
|
||||
@ -2131,12 +2115,13 @@ error:
|
||||
/**
|
||||
* Delete drop queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_flow_delete_drop_queue(struct priv *priv)
|
||||
mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
|
||||
|
||||
if (!fdq)
|
||||
@ -2156,14 +2141,15 @@ priv_flow_delete_drop_queue(struct priv *priv)
|
||||
/**
|
||||
* Remove all flows.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param list
|
||||
* Pointer to a TAILQ flow list.
|
||||
*/
|
||||
void
|
||||
priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
|
||||
mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_flow *flow;
|
||||
|
||||
TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
|
||||
@ -2206,7 +2192,7 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
|
||||
claim_zero(mlx5_glue->destroy_flow
|
||||
(flow->frxq[i].ibv_flow));
|
||||
flow->frxq[i].ibv_flow = NULL;
|
||||
mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
|
||||
mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
|
||||
flow->frxq[i].hrxq = NULL;
|
||||
}
|
||||
DEBUG("Flow %p removed", (void *)flow);
|
||||
@ -2216,8 +2202,8 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
|
||||
/**
|
||||
* Add all flows.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param list
|
||||
* Pointer to a TAILQ flow list.
|
||||
*
|
||||
@ -2225,8 +2211,9 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
|
||||
* 0 on success, a errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
int
|
||||
priv_flow_start(struct priv *priv, struct mlx5_flows *list)
|
||||
mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_flow *flow;
|
||||
|
||||
TAILQ_FOREACH(flow, list, next) {
|
||||
@ -2251,19 +2238,19 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list)
|
||||
if (!flow->frxq[i].ibv_attr)
|
||||
continue;
|
||||
flow->frxq[i].hrxq =
|
||||
mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
|
||||
flow->rss_conf.rss_key_len,
|
||||
hash_rxq_init[i].hash_fields,
|
||||
(*flow->queues),
|
||||
flow->queues_n);
|
||||
mlx5_hrxq_get(dev, flow->rss_conf.rss_key,
|
||||
flow->rss_conf.rss_key_len,
|
||||
hash_rxq_init[i].hash_fields,
|
||||
(*flow->queues),
|
||||
flow->queues_n);
|
||||
if (flow->frxq[i].hrxq)
|
||||
goto flow_create;
|
||||
flow->frxq[i].hrxq =
|
||||
mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
|
||||
flow->rss_conf.rss_key_len,
|
||||
hash_rxq_init[i].hash_fields,
|
||||
(*flow->queues),
|
||||
flow->queues_n);
|
||||
mlx5_hrxq_new(dev, flow->rss_conf.rss_key,
|
||||
flow->rss_conf.rss_key_len,
|
||||
hash_rxq_init[i].hash_fields,
|
||||
(*flow->queues),
|
||||
flow->queues_n);
|
||||
if (!flow->frxq[i].hrxq) {
|
||||
DEBUG("Flow %p cannot be applied",
|
||||
(void *)flow);
|
||||
@ -2293,19 +2280,20 @@ flow_create:
|
||||
/**
|
||||
* Verify the flow list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return the number of flows not released.
|
||||
*/
|
||||
int
|
||||
priv_flow_verify(struct priv *priv)
|
||||
mlx5_flow_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_flow *flow;
|
||||
int ret = 0;
|
||||
|
||||
TAILQ_FOREACH(flow, &priv->flows, next) {
|
||||
DEBUG("%p: flow %p still referenced", (void *)priv,
|
||||
DEBUG("%p: flow %p still referenced", (void *)dev,
|
||||
(void *)flow);
|
||||
++ret;
|
||||
}
|
||||
@ -2386,8 +2374,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
|
||||
action_rss.local.rss_conf = &priv->rss_conf;
|
||||
action_rss.local.num = priv->reta_idx_n;
|
||||
actions[0].conf = (const void *)&action_rss.rss;
|
||||
flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
|
||||
&error);
|
||||
flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
|
||||
actions, &error);
|
||||
if (!flow)
|
||||
return rte_errno;
|
||||
return 0;
|
||||
@ -2427,7 +2415,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
priv_flow_destroy(priv, &priv->flows, flow);
|
||||
mlx5_flow_list_destroy(dev, &priv->flows, flow);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2443,7 +2431,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
priv_flow_flush(priv, &priv->flows);
|
||||
mlx5_flow_list_flush(dev, &priv->flows);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -2460,7 +2448,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
|
||||
* 0 on success, a errno value otherwise and rte_errno is set.
|
||||
*/
|
||||
static int
|
||||
priv_flow_query_count(struct ibv_counter_set *cs,
|
||||
mlx5_flow_query_count(struct ibv_counter_set *cs,
|
||||
struct mlx5_flow_counter_stats *counter_stats,
|
||||
struct rte_flow_query_count *query_count,
|
||||
struct rte_flow_error *error)
|
||||
@ -2510,7 +2498,7 @@ mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
|
||||
int res = EINVAL;
|
||||
|
||||
if (flow->cs) {
|
||||
res = priv_flow_query_count(flow->cs,
|
||||
res = mlx5_flow_query_count(flow->cs,
|
||||
&flow->counter_stats,
|
||||
(struct rte_flow_query_count *)data,
|
||||
error);
|
||||
@ -2555,8 +2543,8 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
|
||||
/**
|
||||
* Convert a flow director filter to a generic flow.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param fdir_filter
|
||||
* Flow director filter to add.
|
||||
* @param attributes
|
||||
@ -2566,10 +2554,11 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
|
||||
* 0 on success, errno value on error.
|
||||
*/
|
||||
static int
|
||||
priv_fdir_filter_convert(struct priv *priv,
|
||||
mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
|
||||
const struct rte_eth_fdir_filter *fdir_filter,
|
||||
struct mlx5_fdir *attributes)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
const struct rte_eth_fdir_input *input = &fdir_filter->input;
|
||||
|
||||
/* Validate queue number. */
|
||||
@ -2741,8 +2730,8 @@ priv_fdir_filter_convert(struct priv *priv,
|
||||
/**
|
||||
* Add new flow director filter and store it in list.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param fdir_filter
|
||||
* Flow director filter to add.
|
||||
*
|
||||
@ -2750,9 +2739,10 @@ priv_fdir_filter_convert(struct priv *priv,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_fdir_filter_add(struct priv *priv,
|
||||
mlx5_fdir_filter_add(struct rte_eth_dev *dev,
|
||||
const struct rte_eth_fdir_filter *fdir_filter)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_fdir attributes = {
|
||||
.attr.group = 0,
|
||||
.l2_mask = {
|
||||
@ -2768,19 +2758,16 @@ priv_fdir_filter_add(struct priv *priv,
|
||||
struct rte_flow *flow;
|
||||
int ret;
|
||||
|
||||
ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
|
||||
ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
|
||||
if (ret)
|
||||
return -ret;
|
||||
ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
|
||||
ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
|
||||
attributes.actions, &error, &parser);
|
||||
if (ret)
|
||||
return -ret;
|
||||
flow = priv_flow_create(priv,
|
||||
&priv->flows,
|
||||
&attributes.attr,
|
||||
attributes.items,
|
||||
attributes.actions,
|
||||
&error);
|
||||
flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
|
||||
attributes.items, attributes.actions,
|
||||
&error);
|
||||
if (flow) {
|
||||
DEBUG("FDIR created %p", (void *)flow);
|
||||
return 0;
|
||||
@ -2791,8 +2778,8 @@ priv_fdir_filter_add(struct priv *priv,
|
||||
/**
|
||||
* Delete specific filter.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param fdir_filter
|
||||
* Filter to be deleted.
|
||||
*
|
||||
@ -2800,9 +2787,10 @@ priv_fdir_filter_add(struct priv *priv,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_fdir_filter_delete(struct priv *priv,
|
||||
mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
|
||||
const struct rte_eth_fdir_filter *fdir_filter)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_fdir attributes = {
|
||||
.attr.group = 0,
|
||||
};
|
||||
@ -2815,10 +2803,10 @@ priv_fdir_filter_delete(struct priv *priv,
|
||||
unsigned int i;
|
||||
int ret;
|
||||
|
||||
ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
|
||||
ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
|
||||
if (ret)
|
||||
return -ret;
|
||||
ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
|
||||
ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
|
||||
attributes.actions, &error, &parser);
|
||||
if (ret)
|
||||
goto exit;
|
||||
@ -2876,7 +2864,7 @@ wrong_flow:
|
||||
continue;
|
||||
}
|
||||
if (flow)
|
||||
priv_flow_destroy(priv, &priv->flows, flow);
|
||||
mlx5_flow_list_destroy(dev, &priv->flows, flow);
|
||||
exit:
|
||||
for (i = 0; i != hash_rxq_init_n; ++i) {
|
||||
if (parser.queue[i].ibv_attr)
|
||||
@ -2888,8 +2876,8 @@ exit:
|
||||
/**
|
||||
* Update queue for specific filter.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param fdir_filter
|
||||
* Filter to be updated.
|
||||
*
|
||||
@ -2897,41 +2885,44 @@ exit:
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_fdir_filter_update(struct priv *priv,
|
||||
mlx5_fdir_filter_update(struct rte_eth_dev *dev,
|
||||
const struct rte_eth_fdir_filter *fdir_filter)
|
||||
{
|
||||
int ret;
|
||||
|
||||
ret = priv_fdir_filter_delete(priv, fdir_filter);
|
||||
ret = mlx5_fdir_filter_delete(dev, fdir_filter);
|
||||
if (ret)
|
||||
return ret;
|
||||
ret = priv_fdir_filter_add(priv, fdir_filter);
|
||||
ret = mlx5_fdir_filter_add(dev, fdir_filter);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush all filters.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
static void
|
||||
priv_fdir_filter_flush(struct priv *priv)
|
||||
mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
|
||||
{
|
||||
priv_flow_flush(priv, &priv->flows);
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
mlx5_flow_list_flush(dev, &priv->flows);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get flow director information.
|
||||
*
|
||||
* @param priv
|
||||
* Private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] fdir_info
|
||||
* Resulting flow director information.
|
||||
*/
|
||||
static void
|
||||
priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
|
||||
mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_eth_fdir_masks *mask =
|
||||
&priv->dev->data->dev_conf.fdir_conf.mask;
|
||||
|
||||
@ -2949,8 +2940,8 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
|
||||
/**
|
||||
* Deal with flow director operations.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param filter_op
|
||||
* Operation to perform.
|
||||
* @param arg
|
||||
@ -2960,8 +2951,10 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
|
||||
mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
|
||||
void *arg)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
enum rte_fdir_mode fdir_mode =
|
||||
priv->dev->data->dev_conf.fdir_conf.mode;
|
||||
int ret = 0;
|
||||
@ -2971,27 +2964,27 @@ priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
|
||||
if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
|
||||
fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
|
||||
ERROR("%p: flow director mode %d not supported",
|
||||
(void *)priv, fdir_mode);
|
||||
(void *)dev, fdir_mode);
|
||||
return EINVAL;
|
||||
}
|
||||
switch (filter_op) {
|
||||
case RTE_ETH_FILTER_ADD:
|
||||
ret = priv_fdir_filter_add(priv, arg);
|
||||
ret = mlx5_fdir_filter_add(dev, arg);
|
||||
break;
|
||||
case RTE_ETH_FILTER_UPDATE:
|
||||
ret = priv_fdir_filter_update(priv, arg);
|
||||
ret = mlx5_fdir_filter_update(dev, arg);
|
||||
break;
|
||||
case RTE_ETH_FILTER_DELETE:
|
||||
ret = priv_fdir_filter_delete(priv, arg);
|
||||
ret = mlx5_fdir_filter_delete(dev, arg);
|
||||
break;
|
||||
case RTE_ETH_FILTER_FLUSH:
|
||||
priv_fdir_filter_flush(priv);
|
||||
mlx5_fdir_filter_flush(dev);
|
||||
break;
|
||||
case RTE_ETH_FILTER_INFO:
|
||||
priv_fdir_info_get(priv, arg);
|
||||
mlx5_fdir_info_get(dev, arg);
|
||||
break;
|
||||
default:
|
||||
DEBUG("%p: unknown operation %u", (void *)priv,
|
||||
DEBUG("%p: unknown operation %u", (void *)dev,
|
||||
filter_op);
|
||||
ret = EINVAL;
|
||||
break;
|
||||
@ -3021,7 +3014,6 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
|
||||
void *arg)
|
||||
{
|
||||
int ret = EINVAL;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
switch (filter_type) {
|
||||
case RTE_ETH_FILTER_GENERIC:
|
||||
@ -3030,7 +3022,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
|
||||
*(const void **)arg = &mlx5_flow_ops;
|
||||
return 0;
|
||||
case RTE_ETH_FILTER_FDIR:
|
||||
ret = priv_fdir_ctrl_func(priv, filter_op, arg);
|
||||
ret = mlx5_fdir_ctrl_func(dev, filter_op, arg);
|
||||
break;
|
||||
default:
|
||||
ERROR("%p: filter type (%d) not supported",
|
||||
|
@ -35,8 +35,8 @@
|
||||
/**
|
||||
* Get MAC address by querying netdevice.
|
||||
*
|
||||
* @param[in] priv
|
||||
* struct priv for the requested device.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] mac
|
||||
* MAC address output buffer.
|
||||
*
|
||||
@ -44,11 +44,11 @@
|
||||
* 0 on success, -1 on failure and errno is set.
|
||||
*/
|
||||
int
|
||||
priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN])
|
||||
mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN])
|
||||
{
|
||||
struct ifreq request;
|
||||
|
||||
if (priv_ifreq(priv, SIOCGIFHWADDR, &request))
|
||||
if (mlx5_ifreq(dev, SIOCGIFHWADDR, &request))
|
||||
return -1;
|
||||
memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
|
||||
return 0;
|
||||
|
@ -83,10 +83,6 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
|
||||
* Register a Memory Region (MR) <-> Memory Pool (MP) association in
|
||||
* txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
|
||||
*
|
||||
* This function should only be called by txq_mp2mr().
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param txq
|
||||
* Pointer to TX queue structure.
|
||||
* @param[in] mp
|
||||
@ -98,29 +94,35 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
|
||||
* mr on success, NULL on failure.
|
||||
*/
|
||||
struct mlx5_mr *
|
||||
priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
|
||||
struct rte_mempool *mp, unsigned int idx)
|
||||
mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
|
||||
unsigned int idx)
|
||||
{
|
||||
struct mlx5_txq_ctrl *txq_ctrl =
|
||||
container_of(txq, struct mlx5_txq_ctrl, txq);
|
||||
struct rte_eth_dev *dev;
|
||||
struct mlx5_mr *mr;
|
||||
|
||||
rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
|
||||
/* Add a new entry, register MR first. */
|
||||
DEBUG("%p: discovered new memory pool \"%s\" (%p)",
|
||||
(void *)txq_ctrl, mp->name, (void *)mp);
|
||||
mr = priv_mr_get(priv, mp);
|
||||
dev = txq_ctrl->priv->dev;
|
||||
mr = mlx5_mr_get(dev, mp);
|
||||
if (mr == NULL) {
|
||||
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
|
||||
DEBUG("Using unregistered mempool 0x%p(%s) in secondary process,"
|
||||
" please create mempool before rte_eth_dev_start()",
|
||||
DEBUG("Using unregistered mempool 0x%p(%s) in "
|
||||
"secondary process, please create mempool before "
|
||||
" rte_eth_dev_start()",
|
||||
(void *)mp, mp->name);
|
||||
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
|
||||
return NULL;
|
||||
}
|
||||
mr = priv_mr_new(priv, mp);
|
||||
mr = mlx5_mr_new(dev, mp);
|
||||
}
|
||||
if (unlikely(mr == NULL)) {
|
||||
DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
|
||||
(void *)txq_ctrl);
|
||||
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
|
||||
return NULL;
|
||||
}
|
||||
if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
|
||||
@ -128,7 +130,7 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
|
||||
DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
|
||||
(void *)txq_ctrl);
|
||||
--idx;
|
||||
priv_mr_release(priv, txq->mp2mr[0]);
|
||||
mlx5_mr_release(txq->mp2mr[0]);
|
||||
memmove(&txq->mp2mr[0], &txq->mp2mr[1],
|
||||
(sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
|
||||
}
|
||||
@ -137,35 +139,6 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
|
||||
DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
|
||||
(void *)txq_ctrl, mp->name, (void *)mp,
|
||||
txq_ctrl->txq.mp2mr[idx]->lkey);
|
||||
return mr;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a Memory Region (MR) <-> Memory Pool (MP) association in
|
||||
* txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
|
||||
*
|
||||
* This function should only be called by txq_mp2mr().
|
||||
*
|
||||
* @param txq
|
||||
* Pointer to TX queue structure.
|
||||
* @param[in] mp
|
||||
* Memory Pool for which a Memory Region lkey must be returned.
|
||||
* @param idx
|
||||
* Index of the next available entry.
|
||||
*
|
||||
* @return
|
||||
* mr on success, NULL on failure.
|
||||
*/
|
||||
struct mlx5_mr*
|
||||
mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
|
||||
unsigned int idx)
|
||||
{
|
||||
struct mlx5_txq_ctrl *txq_ctrl =
|
||||
container_of(txq, struct mlx5_txq_ctrl, txq);
|
||||
struct mlx5_mr *mr;
|
||||
|
||||
rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
|
||||
mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx);
|
||||
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
|
||||
return mr;
|
||||
}
|
||||
@ -225,20 +198,20 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
|
||||
if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
|
||||
data.ret == -1)
|
||||
return;
|
||||
mr = priv_mr_get(priv, mp);
|
||||
mr = mlx5_mr_get(priv->dev, mp);
|
||||
if (mr) {
|
||||
priv_mr_release(priv, mr);
|
||||
mlx5_mr_release(mr);
|
||||
return;
|
||||
}
|
||||
priv_mr_new(priv, mp);
|
||||
mlx5_mr_new(priv->dev, mp);
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a new memory region from the mempool and store it in the memory
|
||||
* region list.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param mp
|
||||
* Pointer to the memory pool to register.
|
||||
*
|
||||
@ -246,8 +219,9 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
|
||||
* The memory region on success.
|
||||
*/
|
||||
struct mlx5_mr *
|
||||
priv_mr_new(struct priv *priv, struct rte_mempool *mp)
|
||||
mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
const struct rte_memseg *ms = rte_eal_get_physmem_layout();
|
||||
uintptr_t start;
|
||||
uintptr_t end;
|
||||
@ -289,7 +263,7 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
|
||||
mr->mp = mp;
|
||||
mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
|
||||
rte_atomic32_inc(&mr->refcnt);
|
||||
DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv,
|
||||
DEBUG("%p: new Memory Region %p refcnt: %d", (void *)dev,
|
||||
(void *)mr, rte_atomic32_read(&mr->refcnt));
|
||||
LIST_INSERT_HEAD(&priv->mr, mr, next);
|
||||
return mr;
|
||||
@ -298,8 +272,8 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
|
||||
/**
|
||||
* Search the memory region object in the memory region list.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param mp
|
||||
* Pointer to the memory pool to register.
|
||||
*
|
||||
@ -307,8 +281,9 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
|
||||
* The memory region on success.
|
||||
*/
|
||||
struct mlx5_mr *
|
||||
priv_mr_get(struct priv *priv, struct rte_mempool *mp)
|
||||
mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_mr *mr;
|
||||
|
||||
assert(mp);
|
||||
@ -335,7 +310,7 @@ priv_mr_get(struct priv *priv, struct rte_mempool *mp)
|
||||
* 0 on success, errno on failure.
|
||||
*/
|
||||
int
|
||||
priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr)
|
||||
mlx5_mr_release(struct mlx5_mr *mr)
|
||||
{
|
||||
assert(mr);
|
||||
DEBUG("Memory Region %p refcnt: %d",
|
||||
@ -352,20 +327,21 @@ priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr)
|
||||
/**
|
||||
* Verify the flow list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
priv_mr_verify(struct priv *priv)
|
||||
mlx5_mr_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int ret = 0;
|
||||
struct mlx5_mr *mr;
|
||||
|
||||
LIST_FOREACH(mr, &priv->mr, next) {
|
||||
DEBUG("%p: mr %p still referenced", (void *)priv,
|
||||
DEBUG("%p: mr %p still referenced", (void *)dev,
|
||||
(void *)mr);
|
||||
++ret;
|
||||
}
|
||||
|
@ -96,8 +96,8 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
|
||||
/**
|
||||
* Allocate/reallocate RETA index table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @praram reta_size
|
||||
* The size of the array to allocate.
|
||||
*
|
||||
@ -105,8 +105,9 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size)
|
||||
mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
void *mem;
|
||||
unsigned int old_size = priv->reta_idx_n;
|
||||
|
||||
@ -126,77 +127,6 @@ priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size)
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Query RETA table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in, out] reta_conf
|
||||
* Pointer to the first RETA configuration structure.
|
||||
* @param reta_size
|
||||
* Number of entries.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_dev_rss_reta_query(struct priv *priv,
|
||||
struct rte_eth_rss_reta_entry64 *reta_conf,
|
||||
unsigned int reta_size)
|
||||
{
|
||||
unsigned int idx;
|
||||
unsigned int i;
|
||||
|
||||
if (!reta_size || reta_size > priv->reta_idx_n)
|
||||
return EINVAL;
|
||||
/* Fill each entry of the table even if its bit is not set. */
|
||||
for (idx = 0, i = 0; (i != reta_size); ++i) {
|
||||
idx = i / RTE_RETA_GROUP_SIZE;
|
||||
reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
|
||||
(*priv->reta_idx)[i];
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update RETA table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] reta_conf
|
||||
* Pointer to the first RETA configuration structure.
|
||||
* @param reta_size
|
||||
* Number of entries.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
static int
|
||||
priv_dev_rss_reta_update(struct priv *priv,
|
||||
struct rte_eth_rss_reta_entry64 *reta_conf,
|
||||
unsigned int reta_size)
|
||||
{
|
||||
unsigned int idx;
|
||||
unsigned int i;
|
||||
unsigned int pos;
|
||||
int ret;
|
||||
|
||||
if (!reta_size)
|
||||
return EINVAL;
|
||||
ret = priv_rss_reta_index_resize(priv, reta_size);
|
||||
if (ret)
|
||||
return ret;
|
||||
for (idx = 0, i = 0; (i != reta_size); ++i) {
|
||||
idx = i / RTE_RETA_GROUP_SIZE;
|
||||
pos = i % RTE_RETA_GROUP_SIZE;
|
||||
if (((reta_conf[idx].mask >> i) & 0x1) == 0)
|
||||
continue;
|
||||
assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
|
||||
(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* DPDK callback to get the RETA indirection table.
|
||||
*
|
||||
@ -215,11 +145,19 @@ mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
|
||||
struct rte_eth_rss_reta_entry64 *reta_conf,
|
||||
uint16_t reta_size)
|
||||
{
|
||||
int ret;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int idx;
|
||||
unsigned int i;
|
||||
|
||||
ret = priv_dev_rss_reta_query(priv, reta_conf, reta_size);
|
||||
return -ret;
|
||||
if (!reta_size || reta_size > priv->reta_idx_n)
|
||||
return -EINVAL;
|
||||
/* Fill each entry of the table even if its bit is not set. */
|
||||
for (idx = 0, i = 0; (i != reta_size); ++i) {
|
||||
idx = i / RTE_RETA_GROUP_SIZE;
|
||||
reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
|
||||
(*priv->reta_idx)[i];
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -242,8 +180,23 @@ mlx5_dev_rss_reta_update(struct rte_eth_dev *dev,
|
||||
{
|
||||
int ret;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int idx;
|
||||
unsigned int i;
|
||||
unsigned int pos;
|
||||
|
||||
ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size);
|
||||
if (!reta_size)
|
||||
return -EINVAL;
|
||||
ret = mlx5_rss_reta_index_resize(dev, reta_size);
|
||||
if (ret)
|
||||
return ret;
|
||||
for (idx = 0, i = 0; (i != reta_size); ++i) {
|
||||
idx = i / RTE_RETA_GROUP_SIZE;
|
||||
pos = i % RTE_RETA_GROUP_SIZE;
|
||||
if (((reta_conf[idx].mask >> i) & 0x1) == 0)
|
||||
continue;
|
||||
assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
|
||||
(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
|
||||
}
|
||||
if (dev->data->dev_started) {
|
||||
mlx5_dev_stop(dev);
|
||||
mlx5_dev_start(dev);
|
||||
|
@ -97,7 +97,7 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
|
||||
(*rxq_ctrl->rxq.elts)[i] = buf;
|
||||
}
|
||||
/* If Rx vector is activated. */
|
||||
if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
|
||||
if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
|
||||
struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
|
||||
struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
|
||||
int j;
|
||||
@ -156,7 +156,7 @@ rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
|
||||
* Some mbuf in the Ring belongs to the application. They cannot be
|
||||
* freed.
|
||||
*/
|
||||
if (rxq_check_vec_support(rxq) > 0) {
|
||||
if (mlx5_rxq_check_vec_support(rxq) > 0) {
|
||||
for (i = 0; i < used; ++i)
|
||||
(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL;
|
||||
rxq->rq_pi = rxq->rq_ci;
|
||||
@ -181,22 +181,23 @@ mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl)
|
||||
{
|
||||
DEBUG("cleaning up %p", (void *)rxq_ctrl);
|
||||
if (rxq_ctrl->ibv)
|
||||
mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv);
|
||||
mlx5_rxq_ibv_release(rxq_ctrl->ibv);
|
||||
memset(rxq_ctrl, 0, sizeof(*rxq_ctrl));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the per-queue supported offloads.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* Supported Rx offloads.
|
||||
*/
|
||||
uint64_t
|
||||
mlx5_priv_get_rx_queue_offloads(struct priv *priv)
|
||||
mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_dev_config *config = &priv->config;
|
||||
uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER |
|
||||
DEV_RX_OFFLOAD_TIMESTAMP |
|
||||
@ -217,13 +218,11 @@ mlx5_priv_get_rx_queue_offloads(struct priv *priv)
|
||||
/**
|
||||
* Returns the per-port supported offloads.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @return
|
||||
* Supported Rx offloads.
|
||||
*/
|
||||
uint64_t
|
||||
mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
|
||||
mlx5_get_rx_port_offloads(void)
|
||||
{
|
||||
uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER;
|
||||
|
||||
@ -233,8 +232,8 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
|
||||
/**
|
||||
* Checks if the per-queue offload configuration is valid.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param offloads
|
||||
* Per-queue offloads configuration.
|
||||
*
|
||||
@ -242,12 +241,11 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
|
||||
* 1 if the configuration is valid, 0 otherwise.
|
||||
*/
|
||||
static int
|
||||
priv_is_rx_queue_offloads_allowed(struct priv *priv, uint64_t offloads)
|
||||
mlx5_is_rx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads)
|
||||
{
|
||||
uint64_t port_offloads = priv->dev->data->dev_conf.rxmode.offloads;
|
||||
uint64_t queue_supp_offloads =
|
||||
mlx5_priv_get_rx_queue_offloads(priv);
|
||||
uint64_t port_supp_offloads = mlx5_priv_get_rx_port_offloads(priv);
|
||||
uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
|
||||
uint64_t queue_supp_offloads = mlx5_get_rx_queue_offloads(dev);
|
||||
uint64_t port_supp_offloads = mlx5_get_rx_port_offloads();
|
||||
|
||||
if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
|
||||
offloads)
|
||||
@ -299,24 +297,24 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
(void *)dev, idx, priv->rxqs_n);
|
||||
return -EOVERFLOW;
|
||||
}
|
||||
if (!priv_is_rx_queue_offloads_allowed(priv, conf->offloads)) {
|
||||
if (!mlx5_is_rx_queue_offloads_allowed(dev, conf->offloads)) {
|
||||
ret = ENOTSUP;
|
||||
ERROR("%p: Rx queue offloads 0x%" PRIx64 " don't match port "
|
||||
"offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64,
|
||||
(void *)dev, conf->offloads,
|
||||
dev->data->dev_conf.rxmode.offloads,
|
||||
(mlx5_priv_get_rx_port_offloads(priv) |
|
||||
mlx5_priv_get_rx_queue_offloads(priv)));
|
||||
(mlx5_get_rx_port_offloads() |
|
||||
mlx5_get_rx_queue_offloads(dev)));
|
||||
goto out;
|
||||
}
|
||||
if (!mlx5_priv_rxq_releasable(priv, idx)) {
|
||||
if (!mlx5_rxq_releasable(dev, idx)) {
|
||||
ret = EBUSY;
|
||||
ERROR("%p: unable to release queue index %u",
|
||||
(void *)dev, idx);
|
||||
goto out;
|
||||
}
|
||||
mlx5_priv_rxq_release(priv, idx);
|
||||
rxq_ctrl = mlx5_priv_rxq_new(priv, idx, desc, socket, conf, mp);
|
||||
mlx5_rxq_release(dev, idx);
|
||||
rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, mp);
|
||||
if (!rxq_ctrl) {
|
||||
ERROR("%p: unable to allocate queue index %u",
|
||||
(void *)dev, idx);
|
||||
@ -347,24 +345,25 @@ mlx5_rx_queue_release(void *dpdk_rxq)
|
||||
return;
|
||||
rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
|
||||
priv = rxq_ctrl->priv;
|
||||
if (!mlx5_priv_rxq_releasable(priv, rxq_ctrl->rxq.stats.idx))
|
||||
if (!mlx5_rxq_releasable(priv->dev, rxq_ctrl->rxq.stats.idx))
|
||||
rte_panic("Rx queue %p is still used by a flow and cannot be"
|
||||
" removed\n", (void *)rxq_ctrl);
|
||||
mlx5_priv_rxq_release(priv, rxq_ctrl->rxq.stats.idx);
|
||||
mlx5_rxq_release(priv->dev, rxq_ctrl->rxq.stats.idx);
|
||||
}
|
||||
|
||||
/**
|
||||
* Allocate queue vector and fill epoll fd list for Rx interrupts.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, negative on failure.
|
||||
*/
|
||||
int
|
||||
priv_rx_intr_vec_enable(struct priv *priv)
|
||||
mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
unsigned int rxqs_n = priv->rxqs_n;
|
||||
unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
|
||||
@ -373,7 +372,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
|
||||
if (!priv->dev->data->dev_conf.intr_conf.rxq)
|
||||
return 0;
|
||||
priv_rx_intr_vec_disable(priv);
|
||||
mlx5_rx_intr_vec_disable(dev);
|
||||
intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
|
||||
if (intr_handle->intr_vec == NULL) {
|
||||
ERROR("failed to allocate memory for interrupt vector,"
|
||||
@ -383,7 +382,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
intr_handle->type = RTE_INTR_HANDLE_EXT;
|
||||
for (i = 0; i != n; ++i) {
|
||||
/* This rxq ibv must not be released in this function. */
|
||||
struct mlx5_rxq_ibv *rxq_ibv = mlx5_priv_rxq_ibv_get(priv, i);
|
||||
struct mlx5_rxq_ibv *rxq_ibv = mlx5_rxq_ibv_get(dev, i);
|
||||
int fd;
|
||||
int flags;
|
||||
int rc;
|
||||
@ -400,7 +399,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
ERROR("too many Rx queues for interrupt vector size"
|
||||
" (%d), Rx interrupts cannot be enabled",
|
||||
RTE_MAX_RXTX_INTR_VEC_ID);
|
||||
priv_rx_intr_vec_disable(priv);
|
||||
mlx5_rx_intr_vec_disable(dev);
|
||||
return -1;
|
||||
}
|
||||
fd = rxq_ibv->channel->fd;
|
||||
@ -409,7 +408,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
if (rc < 0) {
|
||||
ERROR("failed to make Rx interrupt file descriptor"
|
||||
" %d non-blocking for queue index %d", fd, i);
|
||||
priv_rx_intr_vec_disable(priv);
|
||||
mlx5_rx_intr_vec_disable(dev);
|
||||
return -1;
|
||||
}
|
||||
intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
|
||||
@ -417,7 +416,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
count++;
|
||||
}
|
||||
if (!count)
|
||||
priv_rx_intr_vec_disable(priv);
|
||||
mlx5_rx_intr_vec_disable(dev);
|
||||
else
|
||||
intr_handle->nb_efd = count;
|
||||
return 0;
|
||||
@ -426,12 +425,13 @@ priv_rx_intr_vec_enable(struct priv *priv)
|
||||
/**
|
||||
* Clean up Rx interrupts handler.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_rx_intr_vec_disable(struct priv *priv)
|
||||
mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_intr_handle *intr_handle = priv->dev->intr_handle;
|
||||
unsigned int i;
|
||||
unsigned int rxqs_n = priv->rxqs_n;
|
||||
@ -454,7 +454,7 @@ priv_rx_intr_vec_disable(struct priv *priv)
|
||||
*/
|
||||
rxq_data = (*priv->rxqs)[i];
|
||||
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
|
||||
mlx5_priv_rxq_ibv_release(priv, rxq_ctrl->ibv);
|
||||
mlx5_rxq_ibv_release(rxq_ctrl->ibv);
|
||||
}
|
||||
free:
|
||||
rte_intr_free_epoll_fd(intr_handle);
|
||||
@ -516,13 +516,13 @@ mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
|
||||
if (rxq_ctrl->irq) {
|
||||
struct mlx5_rxq_ibv *rxq_ibv;
|
||||
|
||||
rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id);
|
||||
rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
|
||||
if (!rxq_ibv) {
|
||||
ret = EINVAL;
|
||||
goto exit;
|
||||
}
|
||||
mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn);
|
||||
mlx5_priv_rxq_ibv_release(priv, rxq_ibv);
|
||||
mlx5_rxq_ibv_release(rxq_ibv);
|
||||
}
|
||||
exit:
|
||||
if (ret)
|
||||
@ -560,7 +560,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
|
||||
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
|
||||
if (!rxq_ctrl->irq)
|
||||
goto exit;
|
||||
rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id);
|
||||
rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
|
||||
if (!rxq_ibv) {
|
||||
ret = EINVAL;
|
||||
goto exit;
|
||||
@ -574,7 +574,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
|
||||
mlx5_glue->ack_cq_events(rxq_ibv->cq, 1);
|
||||
exit:
|
||||
if (rxq_ibv)
|
||||
mlx5_priv_rxq_ibv_release(priv, rxq_ibv);
|
||||
mlx5_rxq_ibv_release(rxq_ibv);
|
||||
if (ret)
|
||||
WARN("unable to disable interrupt on rx queue %d",
|
||||
rx_queue_id);
|
||||
@ -584,8 +584,8 @@ exit:
|
||||
/**
|
||||
* Create the Rx queue Verbs object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* Queue index in DPDK Rx queue array
|
||||
*
|
||||
@ -593,8 +593,9 @@ exit:
|
||||
* The Verbs object initialised if it can be created.
|
||||
*/
|
||||
struct mlx5_rxq_ibv *
|
||||
mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl =
|
||||
container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
|
||||
@ -629,9 +630,9 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
}
|
||||
tmpl->rxq_ctrl = rxq_ctrl;
|
||||
/* Use the entire RX mempool as the memory region. */
|
||||
tmpl->mr = priv_mr_get(priv, rxq_data->mp);
|
||||
tmpl->mr = mlx5_mr_get(dev, rxq_data->mp);
|
||||
if (!tmpl->mr) {
|
||||
tmpl->mr = priv_mr_new(priv, rxq_data->mp);
|
||||
tmpl->mr = mlx5_mr_new(dev, rxq_data->mp);
|
||||
if (!tmpl->mr) {
|
||||
ERROR("%p: MR creation failure", (void *)rxq_ctrl);
|
||||
goto error;
|
||||
@ -661,7 +662,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
* For vectorized Rx, it must not be doubled in order to
|
||||
* make cq_ci and rq_ci aligned.
|
||||
*/
|
||||
if (rxq_check_vec_support(rxq_data) < 0)
|
||||
if (mlx5_rxq_check_vec_support(rxq_data) < 0)
|
||||
attr.cq.ibv.cqe *= 2;
|
||||
} else if (config->cqe_comp && rxq_data->hw_timestamp) {
|
||||
DEBUG("Rx CQE compression is disabled for HW timestamp");
|
||||
@ -781,7 +782,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
*rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci);
|
||||
DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl);
|
||||
rte_atomic32_inc(&tmpl->refcnt);
|
||||
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
|
||||
LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next);
|
||||
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
|
||||
@ -794,7 +795,7 @@ error:
|
||||
if (tmpl->channel)
|
||||
claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel));
|
||||
if (tmpl->mr)
|
||||
priv_mr_release(priv, tmpl->mr);
|
||||
mlx5_mr_release(tmpl->mr);
|
||||
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
|
||||
return NULL;
|
||||
}
|
||||
@ -802,8 +803,8 @@ error:
|
||||
/**
|
||||
* Get an Rx queue Verbs object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* Queue index in DPDK Rx queue array
|
||||
*
|
||||
@ -811,8 +812,9 @@ error:
|
||||
* The Verbs object if it exists.
|
||||
*/
|
||||
struct mlx5_rxq_ibv *
|
||||
mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl;
|
||||
|
||||
@ -822,9 +824,9 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
return NULL;
|
||||
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
|
||||
if (rxq_ctrl->ibv) {
|
||||
priv_mr_get(priv, rxq_data->mp);
|
||||
mlx5_mr_get(dev, rxq_data->mp);
|
||||
rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
|
||||
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)rxq_ctrl->ibv,
|
||||
rte_atomic32_read(&rxq_ctrl->ibv->refcnt));
|
||||
}
|
||||
@ -834,8 +836,6 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Release an Rx verbs queue object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param rxq_ibv
|
||||
* Verbs Rx queue object.
|
||||
*
|
||||
@ -843,7 +843,7 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
|
||||
mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
|
||||
{
|
||||
int ret;
|
||||
|
||||
@ -851,10 +851,10 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
|
||||
assert(rxq_ibv->wq);
|
||||
assert(rxq_ibv->cq);
|
||||
assert(rxq_ibv->mr);
|
||||
ret = priv_mr_release(priv, rxq_ibv->mr);
|
||||
ret = mlx5_mr_release(rxq_ibv->mr);
|
||||
if (!ret)
|
||||
rxq_ibv->mr = NULL;
|
||||
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("Verbs Rx queue %p: refcnt %d",
|
||||
(void *)rxq_ibv, rte_atomic32_read(&rxq_ibv->refcnt));
|
||||
if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
|
||||
rxq_free_elts(rxq_ibv->rxq_ctrl);
|
||||
@ -873,20 +873,21 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
|
||||
/**
|
||||
* Verify the Verbs Rx queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_ibv_verify(struct priv *priv)
|
||||
mlx5_rxq_ibv_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int ret = 0;
|
||||
struct mlx5_rxq_ibv *rxq_ibv;
|
||||
|
||||
LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) {
|
||||
DEBUG("%p: Verbs Rx queue %p still referenced", (void *)priv,
|
||||
DEBUG("%p: Verbs Rx queue %p still referenced", (void *)dev,
|
||||
(void *)rxq_ibv);
|
||||
++ret;
|
||||
}
|
||||
@ -896,14 +897,11 @@ mlx5_priv_rxq_ibv_verify(struct priv *priv)
|
||||
/**
|
||||
* Return true if a single reference exists on the object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param rxq_ibv
|
||||
* Verbs Rx queue object.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
|
||||
struct mlx5_rxq_ibv *rxq_ibv)
|
||||
mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv)
|
||||
{
|
||||
assert(rxq_ibv);
|
||||
return (rte_atomic32_read(&rxq_ibv->refcnt) == 1);
|
||||
@ -912,8 +910,8 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Create a DPDK Rx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
* @param desc
|
||||
@ -925,11 +923,11 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
|
||||
* A DPDK queue object on success.
|
||||
*/
|
||||
struct mlx5_rxq_ctrl *
|
||||
mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket, const struct rte_eth_rxconf *conf,
|
||||
struct rte_mempool *mp)
|
||||
mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket, const struct rte_eth_rxconf *conf,
|
||||
struct rte_mempool *mp)
|
||||
{
|
||||
struct rte_eth_dev *dev = priv->dev;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_ctrl *tmpl;
|
||||
unsigned int mb_len = rte_pktmbuf_data_room_size(mp);
|
||||
struct mlx5_dev_config *config = &priv->config;
|
||||
@ -1029,7 +1027,7 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
tmpl->rxq.elts =
|
||||
(struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1);
|
||||
rte_atomic32_inc(&tmpl->refcnt);
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
|
||||
LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
|
||||
return tmpl;
|
||||
@ -1041,8 +1039,8 @@ error:
|
||||
/**
|
||||
* Get a Rx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -1050,17 +1048,18 @@ error:
|
||||
* A pointer to the queue if it exists.
|
||||
*/
|
||||
struct mlx5_rxq_ctrl *
|
||||
mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
|
||||
mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl = NULL;
|
||||
|
||||
if ((*priv->rxqs)[idx]) {
|
||||
rxq_ctrl = container_of((*priv->rxqs)[idx],
|
||||
struct mlx5_rxq_ctrl,
|
||||
rxq);
|
||||
mlx5_priv_rxq_ibv_get(priv, idx);
|
||||
mlx5_rxq_ibv_get(dev, idx);
|
||||
rte_atomic32_inc(&rxq_ctrl->refcnt);
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt));
|
||||
}
|
||||
return rxq_ctrl;
|
||||
@ -1069,8 +1068,8 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Release a Rx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -1078,8 +1077,9 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
|
||||
mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl;
|
||||
|
||||
if (!(*priv->rxqs)[idx])
|
||||
@ -1089,11 +1089,11 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
|
||||
if (rxq_ctrl->ibv) {
|
||||
int ret;
|
||||
|
||||
ret = mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv);
|
||||
ret = mlx5_rxq_ibv_release(rxq_ctrl->ibv);
|
||||
if (!ret)
|
||||
rxq_ctrl->ibv = NULL;
|
||||
}
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt));
|
||||
if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
|
||||
LIST_REMOVE(rxq_ctrl, next);
|
||||
@ -1107,8 +1107,8 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Verify if the queue can be released.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -1116,8 +1116,9 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
|
||||
* 1 if the queue can be released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx)
|
||||
mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl;
|
||||
|
||||
if (!(*priv->rxqs)[idx])
|
||||
@ -1129,20 +1130,21 @@ mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Verify the Rx Queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_rxq_verify(struct priv *priv)
|
||||
mlx5_rxq_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl;
|
||||
int ret = 0;
|
||||
|
||||
LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
|
||||
DEBUG("%p: Rx Queue %p still referenced", (void *)priv,
|
||||
DEBUG("%p: Rx Queue %p still referenced", (void *)dev,
|
||||
(void *)rxq_ctrl);
|
||||
++ret;
|
||||
}
|
||||
@ -1152,8 +1154,8 @@ mlx5_priv_rxq_verify(struct priv *priv)
|
||||
/**
|
||||
* Create an indirection table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param queues
|
||||
* Queues entering in the indirection table.
|
||||
* @param queues_n
|
||||
@ -1163,9 +1165,10 @@ mlx5_priv_rxq_verify(struct priv *priv)
|
||||
* A new indirection table.
|
||||
*/
|
||||
struct mlx5_ind_table_ibv *
|
||||
mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
|
||||
uint16_t queues_n)
|
||||
mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, uint16_t queues[],
|
||||
uint16_t queues_n)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_ind_table_ibv *ind_tbl;
|
||||
const unsigned int wq_n = rte_is_power_of_2(queues_n) ?
|
||||
log2above(queues_n) :
|
||||
@ -1179,8 +1182,7 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
|
||||
if (!ind_tbl)
|
||||
return NULL;
|
||||
for (i = 0; i != queues_n; ++i) {
|
||||
struct mlx5_rxq_ctrl *rxq =
|
||||
mlx5_priv_rxq_get(priv, queues[i]);
|
||||
struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev, queues[i]);
|
||||
|
||||
if (!rxq)
|
||||
goto error;
|
||||
@ -1202,20 +1204,20 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
|
||||
goto error;
|
||||
rte_atomic32_inc(&ind_tbl->refcnt);
|
||||
LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
|
||||
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
|
||||
return ind_tbl;
|
||||
error:
|
||||
rte_free(ind_tbl);
|
||||
DEBUG("%p cannot create indirection table", (void *)priv);
|
||||
DEBUG("%p cannot create indirection table", (void *)dev);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get an indirection table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param queues
|
||||
* Queues entering in the indirection table.
|
||||
* @param queues_n
|
||||
@ -1225,9 +1227,10 @@ error:
|
||||
* An indirection table if found.
|
||||
*/
|
||||
struct mlx5_ind_table_ibv *
|
||||
mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
|
||||
uint16_t queues_n)
|
||||
mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, uint16_t queues[],
|
||||
uint16_t queues_n)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_ind_table_ibv *ind_tbl;
|
||||
|
||||
LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
|
||||
@ -1241,10 +1244,10 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
|
||||
unsigned int i;
|
||||
|
||||
rte_atomic32_inc(&ind_tbl->refcnt);
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
|
||||
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
|
||||
for (i = 0; i != ind_tbl->queues_n; ++i)
|
||||
mlx5_priv_rxq_get(priv, ind_tbl->queues[i]);
|
||||
mlx5_rxq_get(dev, ind_tbl->queues[i]);
|
||||
}
|
||||
return ind_tbl;
|
||||
}
|
||||
@ -1252,8 +1255,8 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
|
||||
/**
|
||||
* Release an indirection table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param ind_table
|
||||
* Indirection table to release.
|
||||
*
|
||||
@ -1261,18 +1264,18 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_ind_table_ibv_release(struct priv *priv,
|
||||
struct mlx5_ind_table_ibv *ind_tbl)
|
||||
mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
|
||||
struct mlx5_ind_table_ibv *ind_tbl)
|
||||
{
|
||||
unsigned int i;
|
||||
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
|
||||
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
|
||||
if (rte_atomic32_dec_and_test(&ind_tbl->refcnt))
|
||||
claim_zero(mlx5_glue->destroy_rwq_ind_table
|
||||
(ind_tbl->ind_table));
|
||||
for (i = 0; i != ind_tbl->queues_n; ++i)
|
||||
claim_nonzero(mlx5_priv_rxq_release(priv, ind_tbl->queues[i]));
|
||||
claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
|
||||
if (!rte_atomic32_read(&ind_tbl->refcnt)) {
|
||||
LIST_REMOVE(ind_tbl, next);
|
||||
rte_free(ind_tbl);
|
||||
@ -1284,21 +1287,22 @@ mlx5_priv_ind_table_ibv_release(struct priv *priv,
|
||||
/**
|
||||
* Verify the Rx Queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_ind_table_ibv_verify(struct priv *priv)
|
||||
mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_ind_table_ibv *ind_tbl;
|
||||
int ret = 0;
|
||||
|
||||
LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
|
||||
DEBUG("%p: Verbs indirection table %p still referenced",
|
||||
(void *)priv, (void *)ind_tbl);
|
||||
(void *)dev, (void *)ind_tbl);
|
||||
++ret;
|
||||
}
|
||||
return ret;
|
||||
@ -1307,8 +1311,8 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv)
|
||||
/**
|
||||
* Create an Rx Hash queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param rss_key
|
||||
* RSS key for the Rx hash queue.
|
||||
* @param rss_key_len
|
||||
@ -1325,17 +1329,18 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv)
|
||||
* An hash Rx queue on success.
|
||||
*/
|
||||
struct mlx5_hrxq *
|
||||
mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
|
||||
mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_hrxq *hrxq;
|
||||
struct mlx5_ind_table_ibv *ind_tbl;
|
||||
struct ibv_qp *qp;
|
||||
|
||||
queues_n = hash_fields ? queues_n : 1;
|
||||
ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n);
|
||||
ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
|
||||
if (!ind_tbl)
|
||||
ind_tbl = mlx5_priv_ind_table_ibv_new(priv, queues, queues_n);
|
||||
ind_tbl = mlx5_ind_table_ibv_new(dev, queues, queues_n);
|
||||
if (!ind_tbl)
|
||||
return NULL;
|
||||
qp = mlx5_glue->create_qp_ex
|
||||
@ -1367,11 +1372,11 @@ mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
memcpy(hrxq->rss_key, rss_key, rss_key_len);
|
||||
rte_atomic32_inc(&hrxq->refcnt);
|
||||
LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
|
||||
return hrxq;
|
||||
error:
|
||||
mlx5_priv_ind_table_ibv_release(priv, ind_tbl);
|
||||
mlx5_ind_table_ibv_release(dev, ind_tbl);
|
||||
if (qp)
|
||||
claim_zero(mlx5_glue->destroy_qp(qp));
|
||||
return NULL;
|
||||
@ -1380,8 +1385,8 @@ error:
|
||||
/**
|
||||
* Get an Rx Hash queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param rss_conf
|
||||
* RSS configuration for the Rx hash queue.
|
||||
* @param queues
|
||||
@ -1394,9 +1399,10 @@ error:
|
||||
* An hash Rx queue on success.
|
||||
*/
|
||||
struct mlx5_hrxq *
|
||||
mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
|
||||
mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_hrxq *hrxq;
|
||||
|
||||
queues_n = hash_fields ? queues_n : 1;
|
||||
@ -1409,15 +1415,15 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
continue;
|
||||
if (hrxq->hash_fields != hash_fields)
|
||||
continue;
|
||||
ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n);
|
||||
ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
|
||||
if (!ind_tbl)
|
||||
continue;
|
||||
if (ind_tbl != hrxq->ind_table) {
|
||||
mlx5_priv_ind_table_ibv_release(priv, ind_tbl);
|
||||
mlx5_ind_table_ibv_release(dev, ind_tbl);
|
||||
continue;
|
||||
}
|
||||
rte_atomic32_inc(&hrxq->refcnt);
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
|
||||
return hrxq;
|
||||
}
|
||||
@ -1427,8 +1433,8 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
/**
|
||||
* Release the hash Rx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param hrxq
|
||||
* Pointer to Hash Rx queue to release.
|
||||
*
|
||||
@ -1436,39 +1442,40 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq)
|
||||
mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
|
||||
{
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
|
||||
if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
|
||||
claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
|
||||
mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table);
|
||||
mlx5_ind_table_ibv_release(dev, hrxq->ind_table);
|
||||
LIST_REMOVE(hrxq, next);
|
||||
rte_free(hrxq);
|
||||
return 0;
|
||||
}
|
||||
claim_nonzero(mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table));
|
||||
claim_nonzero(mlx5_ind_table_ibv_release(dev, hrxq->ind_table));
|
||||
return EBUSY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Verify the Rx Queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_hrxq_ibv_verify(struct priv *priv)
|
||||
mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_hrxq *hrxq;
|
||||
int ret = 0;
|
||||
|
||||
LIST_FOREACH(hrxq, &priv->hrxqs, next) {
|
||||
DEBUG("%p: Verbs Hash Rx queue %p still referenced",
|
||||
(void *)priv, (void *)hrxq);
|
||||
(void *)dev, (void *)hrxq);
|
||||
++ret;
|
||||
}
|
||||
return ret;
|
||||
|
@ -1962,27 +1962,25 @@ mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
|
||||
}
|
||||
|
||||
int __attribute__((weak))
|
||||
priv_check_raw_vec_tx_support(struct priv *priv __rte_unused,
|
||||
struct rte_eth_dev *dev __rte_unused)
|
||||
mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
|
||||
{
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
int __attribute__((weak))
|
||||
priv_check_vec_tx_support(struct priv *priv __rte_unused,
|
||||
struct rte_eth_dev *dev __rte_unused)
|
||||
mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
|
||||
{
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
int __attribute__((weak))
|
||||
rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
|
||||
mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
|
||||
{
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
||||
int __attribute__((weak))
|
||||
priv_check_vec_rx_support(struct priv *priv __rte_unused)
|
||||
mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
|
||||
{
|
||||
return -ENOTSUP;
|
||||
}
|
||||
|
@ -215,67 +215,64 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket, const struct rte_eth_rxconf *conf,
|
||||
struct rte_mempool *mp);
|
||||
void mlx5_rx_queue_release(void *dpdk_rxq);
|
||||
int priv_rx_intr_vec_enable(struct priv *priv);
|
||||
void priv_rx_intr_vec_disable(struct priv *priv);
|
||||
int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
|
||||
void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
|
||||
int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
||||
int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
|
||||
struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx);
|
||||
struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv);
|
||||
int mlx5_priv_rxq_ibv_releasable(struct priv *priv,
|
||||
struct mlx5_rxq_ibv *rxq_ibv);
|
||||
int mlx5_priv_rxq_ibv_verify(struct priv *priv);
|
||||
struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *priv, uint16_t idx,
|
||||
uint16_t desc,
|
||||
unsigned int socket,
|
||||
const struct rte_eth_rxconf *conf,
|
||||
struct rte_mempool *mp);
|
||||
struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_rxq_release(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_rxq_verify(struct priv *priv);
|
||||
struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
|
||||
struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
|
||||
int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
|
||||
int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
|
||||
struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
|
||||
uint16_t desc, unsigned int socket,
|
||||
const struct rte_eth_rxconf *conf,
|
||||
struct rte_mempool *mp);
|
||||
struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_rxq_verify(struct rte_eth_dev *dev);
|
||||
int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
|
||||
struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_new(struct priv *priv,
|
||||
uint16_t queues[],
|
||||
uint16_t queues_n);
|
||||
struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *priv,
|
||||
uint16_t queues[],
|
||||
uint16_t queues_n);
|
||||
int mlx5_priv_ind_table_ibv_release(struct priv *priv,
|
||||
struct mlx5_ind_table_ibv *ind_tbl);
|
||||
int mlx5_priv_ind_table_ibv_verify(struct priv *priv);
|
||||
struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key,
|
||||
uint8_t rss_key_len, uint64_t hash_fields,
|
||||
uint16_t queues[], uint16_t queues_n);
|
||||
struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key,
|
||||
uint8_t rss_key_len, uint64_t hash_fields,
|
||||
uint16_t queues[], uint16_t queues_n);
|
||||
int mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq);
|
||||
int mlx5_priv_hrxq_ibv_verify(struct priv *priv);
|
||||
uint64_t mlx5_priv_get_rx_port_offloads(struct priv *priv);
|
||||
uint64_t mlx5_priv_get_rx_queue_offloads(struct priv *priv);
|
||||
struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev,
|
||||
uint16_t queues[],
|
||||
uint16_t queues_n);
|
||||
struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
|
||||
uint16_t queues[],
|
||||
uint16_t queues_n);
|
||||
int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
|
||||
struct mlx5_ind_table_ibv *ind_tbl);
|
||||
int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
|
||||
struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key,
|
||||
uint8_t rss_key_len, uint64_t hash_fields,
|
||||
uint16_t queues[], uint16_t queues_n);
|
||||
struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key,
|
||||
uint8_t rss_key_len, uint64_t hash_fields,
|
||||
uint16_t queues[], uint16_t queues_n);
|
||||
int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
|
||||
int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
|
||||
uint64_t mlx5_get_rx_port_offloads(void);
|
||||
uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_txq.c */
|
||||
|
||||
int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket, const struct rte_eth_txconf *conf);
|
||||
void mlx5_tx_queue_release(void *dpdk_txq);
|
||||
int priv_tx_uar_remap(struct priv *priv, int fd);
|
||||
struct mlx5_txq_ibv *mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx);
|
||||
struct mlx5_txq_ibv *mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv);
|
||||
int mlx5_priv_txq_ibv_releasable(struct priv *priv,
|
||||
struct mlx5_txq_ibv *txq_ibv);
|
||||
int mlx5_priv_txq_ibv_verify(struct priv *priv);
|
||||
struct mlx5_txq_ctrl *mlx5_priv_txq_new(struct priv *priv, uint16_t idx,
|
||||
uint16_t desc, unsigned int socket,
|
||||
const struct rte_eth_txconf *conf);
|
||||
struct mlx5_txq_ctrl *mlx5_priv_txq_get(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_txq_release(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx);
|
||||
int mlx5_priv_txq_verify(struct priv *priv);
|
||||
int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd);
|
||||
struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
|
||||
struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
|
||||
int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv);
|
||||
int mlx5_txq_ibv_verify(struct rte_eth_dev *dev);
|
||||
struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
|
||||
uint16_t desc, unsigned int socket,
|
||||
const struct rte_eth_txconf *conf);
|
||||
struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
|
||||
int mlx5_txq_verify(struct rte_eth_dev *dev);
|
||||
void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
|
||||
uint64_t mlx5_priv_get_tx_port_offloads(struct priv *priv);
|
||||
uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
|
||||
|
||||
/* mlx5_rxtx.c */
|
||||
|
||||
@ -299,26 +296,22 @@ int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
|
||||
int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
|
||||
|
||||
/* Vectorized version of mlx5_rxtx.c */
|
||||
|
||||
int priv_check_raw_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev);
|
||||
int priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev);
|
||||
int rxq_check_vec_support(struct mlx5_rxq_data *rxq);
|
||||
int priv_check_vec_rx_support(struct priv *priv);
|
||||
int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev);
|
||||
int mlx5_check_vec_tx_support(struct rte_eth_dev *dev);
|
||||
int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
|
||||
int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
|
||||
uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
|
||||
uint16_t pkts_n);
|
||||
uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
|
||||
uint16_t pkts_n);
|
||||
uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
|
||||
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
|
||||
uint16_t pkts_n);
|
||||
|
||||
/* mlx5_mr.c */
|
||||
|
||||
void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg);
|
||||
struct mlx5_mr *priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
|
||||
struct rte_mempool *mp, unsigned int idx);
|
||||
struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq,
|
||||
struct rte_mempool *mp,
|
||||
unsigned int idx);
|
||||
struct rte_mempool *mp, unsigned int idx);
|
||||
|
||||
#ifndef NDEBUG
|
||||
/**
|
||||
|
@ -223,17 +223,14 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
|
||||
/**
|
||||
* Check Tx queue flags are set for raw vectorized Tx.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to rte_eth_dev structure.
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 1 if supported, negative errno value if not.
|
||||
*/
|
||||
int __attribute__((cold))
|
||||
priv_check_raw_vec_tx_support(__rte_unused struct priv *priv,
|
||||
struct rte_eth_dev *dev)
|
||||
mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev)
|
||||
{
|
||||
uint64_t offloads = dev->data->dev_conf.txmode.offloads;
|
||||
|
||||
@ -246,17 +243,16 @@ priv_check_raw_vec_tx_support(__rte_unused struct priv *priv,
|
||||
/**
|
||||
* Check a device can support vectorized TX.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to rte_eth_dev structure.
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 1 if supported, negative errno value if not.
|
||||
*/
|
||||
int __attribute__((cold))
|
||||
priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev)
|
||||
mlx5_check_vec_tx_support(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
uint64_t offloads = dev->data->dev_conf.txmode.offloads;
|
||||
|
||||
if (!priv->config.tx_vec_en ||
|
||||
@ -277,7 +273,7 @@ priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev)
|
||||
* 1 if supported, negative errno value if not.
|
||||
*/
|
||||
int __attribute__((cold))
|
||||
rxq_check_vec_support(struct mlx5_rxq_data *rxq)
|
||||
mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
|
||||
{
|
||||
struct mlx5_rxq_ctrl *ctrl =
|
||||
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
|
||||
@ -290,15 +286,16 @@ rxq_check_vec_support(struct mlx5_rxq_data *rxq)
|
||||
/**
|
||||
* Check a device can support vectorized RX.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 1 if supported, negative errno value if not.
|
||||
*/
|
||||
int __attribute__((cold))
|
||||
priv_check_vec_rx_support(struct priv *priv)
|
||||
mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
uint16_t i;
|
||||
|
||||
if (!priv->config.rx_vec_en)
|
||||
@ -309,7 +306,7 @@ priv_check_vec_rx_support(struct priv *priv)
|
||||
|
||||
if (!rxq)
|
||||
continue;
|
||||
if (rxq_check_vec_support(rxq) < 0)
|
||||
if (mlx5_rxq_check_vec_support(rxq) < 0)
|
||||
break;
|
||||
}
|
||||
if (i != priv->rxqs_n)
|
||||
|
@ -18,15 +18,16 @@
|
||||
/**
|
||||
* Initialise the socket to communicate with the secondary process
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_socket_init(struct priv *priv)
|
||||
mlx5_socket_init(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct sockaddr_un sun = {
|
||||
.sun_family = AF_UNIX,
|
||||
};
|
||||
@ -79,15 +80,17 @@ out:
|
||||
/**
|
||||
* Un-Initialise the socket to communicate with the secondary process
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_socket_uninit(struct priv *priv)
|
||||
mlx5_socket_uninit(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket);
|
||||
claim_zero(close(priv->primary_socket));
|
||||
priv->primary_socket = 0;
|
||||
@ -98,12 +101,13 @@ priv_socket_uninit(struct priv *priv)
|
||||
/**
|
||||
* Handle socket interrupts.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_socket_handle(struct priv *priv)
|
||||
mlx5_socket_handle(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int conn_sock;
|
||||
int ret = 0;
|
||||
struct cmsghdr *cmsg = NULL;
|
||||
@ -179,15 +183,16 @@ out:
|
||||
/**
|
||||
* Connect to the primary process.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet structure.
|
||||
*
|
||||
* @return
|
||||
* fd on success, negative errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_socket_connect(struct priv *priv)
|
||||
mlx5_socket_connect(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct sockaddr_un sun = {
|
||||
.sun_family = AF_UNIX,
|
||||
};
|
||||
|
@ -122,8 +122,8 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
|
||||
/**
|
||||
* Read device counters table.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] stats
|
||||
* Counters table output buffer.
|
||||
*
|
||||
@ -131,8 +131,9 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
|
||||
* 0 on success and stats is filled, negative on error.
|
||||
*/
|
||||
static int
|
||||
priv_read_dev_counters(struct priv *priv, uint64_t *stats)
|
||||
mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
unsigned int i;
|
||||
struct ifreq ifr;
|
||||
@ -143,7 +144,7 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats)
|
||||
et_stats->cmd = ETHTOOL_GSTATS;
|
||||
et_stats->n_stats = xstats_ctrl->stats_n;
|
||||
ifr.ifr_data = (caddr_t)et_stats;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
|
||||
WARN("unable to read statistic values from device");
|
||||
return -1;
|
||||
}
|
||||
@ -173,20 +174,20 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats)
|
||||
/**
|
||||
* Query the number of statistics provided by ETHTOOL.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* Number of statistics on success, -1 on error.
|
||||
*/
|
||||
static int
|
||||
priv_ethtool_get_stats_n(struct priv *priv) {
|
||||
mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) {
|
||||
struct ethtool_drvinfo drvinfo;
|
||||
struct ifreq ifr;
|
||||
|
||||
drvinfo.cmd = ETHTOOL_GDRVINFO;
|
||||
ifr.ifr_data = (caddr_t)&drvinfo;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
|
||||
WARN("unable to query number of statistics");
|
||||
return -1;
|
||||
}
|
||||
@ -196,12 +197,13 @@ priv_ethtool_get_stats_n(struct priv *priv) {
|
||||
/**
|
||||
* Init the structures to read device counters.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*/
|
||||
void
|
||||
priv_xstats_init(struct priv *priv)
|
||||
mlx5_xstats_init(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
unsigned int i;
|
||||
unsigned int j;
|
||||
@ -210,7 +212,7 @@ priv_xstats_init(struct priv *priv)
|
||||
unsigned int dev_stats_n;
|
||||
unsigned int str_sz;
|
||||
|
||||
dev_stats_n = priv_ethtool_get_stats_n(priv);
|
||||
dev_stats_n = mlx5_ethtool_get_stats_n(dev);
|
||||
if (dev_stats_n < 1) {
|
||||
WARN("no extended statistics available");
|
||||
return;
|
||||
@ -229,7 +231,7 @@ priv_xstats_init(struct priv *priv)
|
||||
strings->string_set = ETH_SS_STATS;
|
||||
strings->len = dev_stats_n;
|
||||
ifr.ifr_data = (caddr_t)strings;
|
||||
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
|
||||
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
|
||||
WARN("unable to get statistic names");
|
||||
goto free;
|
||||
}
|
||||
@ -258,60 +260,54 @@ priv_xstats_init(struct priv *priv)
|
||||
}
|
||||
/* Copy to base at first time. */
|
||||
assert(xstats_n <= MLX5_MAX_XSTATS);
|
||||
priv_read_dev_counters(priv, xstats_ctrl->base);
|
||||
mlx5_read_dev_counters(dev, xstats_ctrl->base);
|
||||
free:
|
||||
rte_free(strings);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get device extended statistics.
|
||||
* DPDK callback to get extended device statistics.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param[out] stats
|
||||
* Pointer to rte extended stats table.
|
||||
* @param n
|
||||
* The size of the stats table.
|
||||
*
|
||||
* @return
|
||||
* Number of extended stats on success and stats is filled,
|
||||
* negative on error.
|
||||
*/
|
||||
static int
|
||||
priv_xstats_get(struct priv *priv, struct rte_eth_xstat *stats)
|
||||
int
|
||||
mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
|
||||
unsigned int n)
|
||||
{
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
unsigned int n = xstats_n;
|
||||
uint64_t counters[n];
|
||||
int ret = 0;
|
||||
|
||||
if (priv_read_dev_counters(priv, counters) < 0)
|
||||
return -1;
|
||||
for (i = 0; i != xstats_n; ++i) {
|
||||
stats[i].id = i;
|
||||
stats[i].value = (counters[i] - xstats_ctrl->base[i]);
|
||||
if (n >= xstats_n && stats) {
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
int stats_n;
|
||||
|
||||
stats_n = mlx5_ethtool_get_stats_n(dev);
|
||||
if (stats_n < 0)
|
||||
return -1;
|
||||
if (xstats_ctrl->stats_n != stats_n)
|
||||
mlx5_xstats_init(dev);
|
||||
ret = mlx5_read_dev_counters(dev, counters);
|
||||
if (ret)
|
||||
return ret;
|
||||
for (i = 0; i != xstats_n; ++i) {
|
||||
stats[i].id = i;
|
||||
stats[i].value = (counters[i] - xstats_ctrl->base[i]);
|
||||
}
|
||||
}
|
||||
return n;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset device extended statistics.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
*/
|
||||
static void
|
||||
priv_xstats_reset(struct priv *priv)
|
||||
{
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
unsigned int i;
|
||||
unsigned int n = xstats_n;
|
||||
uint64_t counters[n];
|
||||
|
||||
if (priv_read_dev_counters(priv, counters) < 0)
|
||||
return;
|
||||
for (i = 0; i != n; ++i)
|
||||
xstats_ctrl->base[i] = counters[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* DPDK callback to get device statistics.
|
||||
*
|
||||
@ -408,41 +404,6 @@ mlx5_stats_reset(struct rte_eth_dev *dev)
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* DPDK callback to get extended device statistics.
|
||||
*
|
||||
* @param dev
|
||||
* Pointer to Ethernet device structure.
|
||||
* @param[out] stats
|
||||
* Stats table output buffer.
|
||||
* @param n
|
||||
* The size of the stats table.
|
||||
*
|
||||
* @return
|
||||
* Number of xstats on success, negative on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_xstats_get(struct rte_eth_dev *dev,
|
||||
struct rte_eth_xstat *stats, unsigned int n)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int ret = xstats_n;
|
||||
|
||||
if (n >= xstats_n && stats) {
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
int stats_n;
|
||||
|
||||
stats_n = priv_ethtool_get_stats_n(priv);
|
||||
if (stats_n < 0) {
|
||||
return -1;
|
||||
}
|
||||
if (xstats_ctrl->stats_n != stats_n)
|
||||
priv_xstats_init(priv);
|
||||
ret = priv_xstats_get(priv, stats);
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* DPDK callback to clear device extended statistics.
|
||||
*
|
||||
@ -455,13 +416,19 @@ mlx5_xstats_reset(struct rte_eth_dev *dev)
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
|
||||
int stats_n;
|
||||
unsigned int i;
|
||||
unsigned int n = xstats_n;
|
||||
uint64_t counters[n];
|
||||
|
||||
stats_n = priv_ethtool_get_stats_n(priv);
|
||||
stats_n = mlx5_ethtool_get_stats_n(dev);
|
||||
if (stats_n < 0)
|
||||
return;
|
||||
if (xstats_ctrl->stats_n != stats_n)
|
||||
priv_xstats_init(priv);
|
||||
priv_xstats_reset(priv);
|
||||
mlx5_xstats_init(dev);
|
||||
if (mlx5_read_dev_counters(dev, counters) < 0)
|
||||
return;
|
||||
for (i = 0; i != n; ++i)
|
||||
xstats_ctrl->base[i] = counters[i];
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -21,12 +21,13 @@
|
||||
* Pointer to Ethernet device structure.
|
||||
*/
|
||||
static void
|
||||
priv_txq_stop(struct priv *priv)
|
||||
mlx5_txq_stop(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i != priv->txqs_n; ++i)
|
||||
mlx5_priv_txq_release(priv, i);
|
||||
mlx5_txq_release(dev, i);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -39,8 +40,9 @@ priv_txq_stop(struct priv *priv)
|
||||
* 0 on success, errno on error.
|
||||
*/
|
||||
static int
|
||||
priv_txq_start(struct priv *priv)
|
||||
mlx5_txq_start(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
int ret = 0;
|
||||
|
||||
@ -48,28 +50,28 @@ priv_txq_start(struct priv *priv)
|
||||
for (i = 0; i != priv->txqs_n; ++i) {
|
||||
unsigned int idx = 0;
|
||||
struct mlx5_mr *mr;
|
||||
struct mlx5_txq_ctrl *txq_ctrl = mlx5_priv_txq_get(priv, i);
|
||||
struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
|
||||
|
||||
if (!txq_ctrl)
|
||||
continue;
|
||||
LIST_FOREACH(mr, &priv->mr, next) {
|
||||
priv_txq_mp2mr_reg(priv, &txq_ctrl->txq, mr->mp, idx++);
|
||||
mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++);
|
||||
if (idx == MLX5_PMD_TX_MP_CACHE)
|
||||
break;
|
||||
}
|
||||
txq_alloc_elts(txq_ctrl);
|
||||
txq_ctrl->ibv = mlx5_priv_txq_ibv_new(priv, i);
|
||||
txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
|
||||
if (!txq_ctrl->ibv) {
|
||||
ret = ENOMEM;
|
||||
goto error;
|
||||
}
|
||||
}
|
||||
ret = priv_tx_uar_remap(priv, priv->ctx->cmd_fd);
|
||||
ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
|
||||
if (ret)
|
||||
goto error;
|
||||
return ret;
|
||||
error:
|
||||
priv_txq_stop(priv);
|
||||
mlx5_txq_stop(dev);
|
||||
return ret;
|
||||
}
|
||||
|
||||
@ -80,12 +82,13 @@ error:
|
||||
* Pointer to Ethernet device structure.
|
||||
*/
|
||||
static void
|
||||
priv_rxq_stop(struct priv *priv)
|
||||
mlx5_rxq_stop(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
|
||||
for (i = 0; i != priv->rxqs_n; ++i)
|
||||
mlx5_priv_rxq_release(priv, i);
|
||||
mlx5_rxq_release(dev, i);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -98,20 +101,21 @@ priv_rxq_stop(struct priv *priv)
|
||||
* 0 on success, errno on error.
|
||||
*/
|
||||
static int
|
||||
priv_rxq_start(struct priv *priv)
|
||||
mlx5_rxq_start(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
int ret = 0;
|
||||
|
||||
for (i = 0; i != priv->rxqs_n; ++i) {
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_priv_rxq_get(priv, i);
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
|
||||
|
||||
if (!rxq_ctrl)
|
||||
continue;
|
||||
ret = rxq_alloc_elts(rxq_ctrl);
|
||||
if (ret)
|
||||
goto error;
|
||||
rxq_ctrl->ibv = mlx5_priv_rxq_ibv_new(priv, i);
|
||||
rxq_ctrl->ibv = mlx5_rxq_ibv_new(dev, i);
|
||||
if (!rxq_ctrl->ibv) {
|
||||
ret = ENOMEM;
|
||||
goto error;
|
||||
@ -119,7 +123,7 @@ priv_rxq_start(struct priv *priv)
|
||||
}
|
||||
return -ret;
|
||||
error:
|
||||
priv_rxq_stop(priv);
|
||||
mlx5_rxq_stop(dev);
|
||||
return -ret;
|
||||
}
|
||||
|
||||
@ -142,7 +146,7 @@ mlx5_dev_start(struct rte_eth_dev *dev)
|
||||
int err;
|
||||
|
||||
dev->data->dev_started = 1;
|
||||
err = priv_flow_create_drop_queue(priv);
|
||||
err = mlx5_flow_create_drop_queue(dev);
|
||||
if (err) {
|
||||
ERROR("%p: Drop queue allocation failed: %s",
|
||||
(void *)dev, strerror(err));
|
||||
@ -150,46 +154,46 @@ mlx5_dev_start(struct rte_eth_dev *dev)
|
||||
}
|
||||
DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
|
||||
rte_mempool_walk(mlx5_mp2mr_iter, priv);
|
||||
err = priv_txq_start(priv);
|
||||
err = mlx5_txq_start(dev);
|
||||
if (err) {
|
||||
ERROR("%p: TXQ allocation failed: %s",
|
||||
(void *)dev, strerror(err));
|
||||
goto error;
|
||||
}
|
||||
err = priv_rxq_start(priv);
|
||||
err = mlx5_rxq_start(dev);
|
||||
if (err) {
|
||||
ERROR("%p: RXQ allocation failed: %s",
|
||||
(void *)dev, strerror(err));
|
||||
goto error;
|
||||
}
|
||||
err = priv_rx_intr_vec_enable(priv);
|
||||
err = mlx5_rx_intr_vec_enable(dev);
|
||||
if (err) {
|
||||
ERROR("%p: RX interrupt vector creation failed",
|
||||
(void *)priv);
|
||||
goto error;
|
||||
}
|
||||
priv_xstats_init(priv);
|
||||
mlx5_xstats_init(dev);
|
||||
/* Update link status and Tx/Rx callbacks for the first time. */
|
||||
memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link));
|
||||
INFO("Forcing port %u link to be up", dev->data->port_id);
|
||||
err = priv_force_link_status_change(priv, ETH_LINK_UP);
|
||||
err = mlx5_force_link_status_change(dev, ETH_LINK_UP);
|
||||
if (err) {
|
||||
DEBUG("Failed to set port %u link to be up",
|
||||
dev->data->port_id);
|
||||
goto error;
|
||||
}
|
||||
priv_dev_interrupt_handler_install(priv, dev);
|
||||
mlx5_dev_interrupt_handler_install(dev);
|
||||
return 0;
|
||||
error:
|
||||
/* Rollback. */
|
||||
dev->data->dev_started = 0;
|
||||
for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
|
||||
priv_mr_release(priv, mr);
|
||||
priv_flow_stop(priv, &priv->flows);
|
||||
priv_dev_traffic_disable(priv, dev);
|
||||
priv_txq_stop(priv);
|
||||
priv_rxq_stop(priv);
|
||||
priv_flow_delete_drop_queue(priv);
|
||||
mlx5_mr_release(mr);
|
||||
mlx5_flow_stop(dev, &priv->flows);
|
||||
mlx5_traffic_disable(dev);
|
||||
mlx5_txq_stop(dev);
|
||||
mlx5_rxq_stop(dev);
|
||||
mlx5_flow_delete_drop_queue(dev);
|
||||
return err;
|
||||
}
|
||||
|
||||
@ -214,21 +218,21 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
|
||||
rte_wmb();
|
||||
usleep(1000 * priv->rxqs_n);
|
||||
DEBUG("%p: cleaning up and destroying hash RX queues", (void *)dev);
|
||||
priv_flow_stop(priv, &priv->flows);
|
||||
priv_dev_traffic_disable(priv, dev);
|
||||
priv_rx_intr_vec_disable(priv);
|
||||
priv_dev_interrupt_handler_uninstall(priv, dev);
|
||||
priv_txq_stop(priv);
|
||||
priv_rxq_stop(priv);
|
||||
mlx5_flow_stop(dev, &priv->flows);
|
||||
mlx5_traffic_disable(dev);
|
||||
mlx5_rx_intr_vec_disable(dev);
|
||||
mlx5_dev_interrupt_handler_uninstall(dev);
|
||||
mlx5_txq_stop(dev);
|
||||
mlx5_rxq_stop(dev);
|
||||
for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
|
||||
priv_mr_release(priv, mr);
|
||||
priv_flow_delete_drop_queue(priv);
|
||||
mlx5_mr_release(mr);
|
||||
mlx5_flow_delete_drop_queue(dev);
|
||||
}
|
||||
|
||||
/**
|
||||
* Enable traffic flows configured by control plane
|
||||
*
|
||||
* @param priv
|
||||
* @param dev
|
||||
* Pointer to Ethernet device private data.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device structure.
|
||||
@ -237,8 +241,9 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
|
||||
* 0 on success.
|
||||
*/
|
||||
int
|
||||
priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev)
|
||||
mlx5_traffic_enable(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct rte_flow_item_eth bcast = {
|
||||
.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
|
||||
};
|
||||
@ -356,40 +361,18 @@ error:
|
||||
/**
|
||||
* Disable traffic flows configured by control plane
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to Ethernet device private data.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device structure.
|
||||
* Pointer to Ethernet device private data.
|
||||
*
|
||||
* @return
|
||||
* 0 on success.
|
||||
*/
|
||||
int
|
||||
priv_dev_traffic_disable(struct priv *priv,
|
||||
struct rte_eth_dev *dev __rte_unused)
|
||||
mlx5_traffic_disable(struct rte_eth_dev *dev)
|
||||
{
|
||||
priv_flow_flush(priv, &priv->ctrl_flows);
|
||||
return 0;
|
||||
}
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
/**
|
||||
* Restart traffic flows configured by control plane
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to Ethernet device private data.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device structure.
|
||||
*
|
||||
* @return
|
||||
* 0 on success.
|
||||
*/
|
||||
int
|
||||
priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
|
||||
{
|
||||
if (dev->data->dev_started) {
|
||||
priv_dev_traffic_disable(priv, dev);
|
||||
priv_dev_traffic_enable(priv, dev);
|
||||
}
|
||||
mlx5_flow_list_flush(dev, &priv->ctrl_flows);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -397,7 +380,7 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
|
||||
* Restart traffic flows configured by control plane
|
||||
*
|
||||
* @param dev
|
||||
* Pointer to Ethernet device structure.
|
||||
* Pointer to Ethernet device private data.
|
||||
*
|
||||
* @return
|
||||
* 0 on success.
|
||||
@ -405,8 +388,9 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
|
||||
int
|
||||
mlx5_traffic_restart(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
|
||||
priv_dev_traffic_restart(priv, dev);
|
||||
if (dev->data->dev_started) {
|
||||
mlx5_traffic_disable(dev);
|
||||
mlx5_traffic_enable(dev);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
@ -91,15 +91,16 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl)
|
||||
/**
|
||||
* Returns the per-port supported offloads.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* Supported Tx offloads.
|
||||
*/
|
||||
uint64_t
|
||||
mlx5_priv_get_tx_port_offloads(struct priv *priv)
|
||||
mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS |
|
||||
DEV_TX_OFFLOAD_VLAN_INSERT);
|
||||
struct mlx5_dev_config *config = &priv->config;
|
||||
@ -123,8 +124,8 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv)
|
||||
/**
|
||||
* Checks if the per-queue offload configuration is valid.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param offloads
|
||||
* Per-queue offloads configuration.
|
||||
*
|
||||
@ -132,10 +133,10 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv)
|
||||
* 1 if the configuration is valid, 0 otherwise.
|
||||
*/
|
||||
static int
|
||||
priv_is_tx_queue_offloads_allowed(struct priv *priv, uint64_t offloads)
|
||||
mlx5_is_tx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads)
|
||||
{
|
||||
uint64_t port_offloads = priv->dev->data->dev_conf.txmode.offloads;
|
||||
uint64_t port_supp_offloads = mlx5_priv_get_tx_port_offloads(priv);
|
||||
uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
|
||||
uint64_t port_supp_offloads = mlx5_get_tx_port_offloads(dev);
|
||||
|
||||
/* There are no Tx offloads which are per queue. */
|
||||
if ((offloads & port_supp_offloads) != offloads)
|
||||
@ -177,13 +178,13 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
* use the old API.
|
||||
*/
|
||||
if (!!(conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) &&
|
||||
!priv_is_tx_queue_offloads_allowed(priv, conf->offloads)) {
|
||||
!mlx5_is_tx_queue_offloads_allowed(dev, conf->offloads)) {
|
||||
ret = ENOTSUP;
|
||||
ERROR("%p: Tx queue offloads 0x%" PRIx64 " don't match port "
|
||||
"offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64,
|
||||
(void *)dev, conf->offloads,
|
||||
dev->data->dev_conf.txmode.offloads,
|
||||
mlx5_priv_get_tx_port_offloads(priv));
|
||||
mlx5_get_tx_port_offloads(dev));
|
||||
goto out;
|
||||
}
|
||||
if (desc <= MLX5_TX_COMP_THRESH) {
|
||||
@ -206,14 +207,14 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
(void *)dev, idx, priv->txqs_n);
|
||||
return -EOVERFLOW;
|
||||
}
|
||||
if (!mlx5_priv_txq_releasable(priv, idx)) {
|
||||
if (!mlx5_txq_releasable(dev, idx)) {
|
||||
ret = EBUSY;
|
||||
ERROR("%p: unable to release queue index %u",
|
||||
(void *)dev, idx);
|
||||
goto out;
|
||||
}
|
||||
mlx5_priv_txq_release(priv, idx);
|
||||
txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf);
|
||||
mlx5_txq_release(dev, idx);
|
||||
txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf);
|
||||
if (!txq_ctrl) {
|
||||
ERROR("%p: unable to allocate queue index %u",
|
||||
(void *)dev, idx);
|
||||
@ -249,7 +250,7 @@ mlx5_tx_queue_release(void *dpdk_txq)
|
||||
if ((*priv->txqs)[i] == txq) {
|
||||
DEBUG("%p: removing TX queue %p from list",
|
||||
(void *)priv->dev, (void *)txq_ctrl);
|
||||
mlx5_priv_txq_release(priv, i);
|
||||
mlx5_txq_release(priv->dev, i);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -260,8 +261,8 @@ mlx5_tx_queue_release(void *dpdk_txq)
|
||||
* Both primary and secondary process do mmap to make UAR address
|
||||
* aligned.
|
||||
*
|
||||
* @param[in] priv
|
||||
* Pointer to private structure.
|
||||
* @param[in] dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param fd
|
||||
* Verbs file descriptor to map UAR pages.
|
||||
*
|
||||
@ -269,8 +270,9 @@ mlx5_tx_queue_release(void *dpdk_txq)
|
||||
* 0 on success, errno value on failure.
|
||||
*/
|
||||
int
|
||||
priv_tx_uar_remap(struct priv *priv, int fd)
|
||||
mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i, j;
|
||||
uintptr_t pages[priv->txqs_n];
|
||||
unsigned int pages_n = 0;
|
||||
@ -356,8 +358,8 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst)
|
||||
/**
|
||||
* Create the Tx queue Verbs object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* Queue index in DPDK Rx queue array
|
||||
*
|
||||
@ -365,8 +367,9 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst)
|
||||
* The Verbs object initialised if it can be created.
|
||||
*/
|
||||
struct mlx5_txq_ibv *
|
||||
mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
|
||||
struct mlx5_txq_ctrl *txq_ctrl =
|
||||
container_of(txq_data, struct mlx5_txq_ctrl, txq);
|
||||
@ -383,7 +386,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
struct mlx5dv_cq cq_info;
|
||||
struct mlx5dv_obj obj;
|
||||
const int desc = 1 << txq_data->elts_n;
|
||||
eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev);
|
||||
eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev);
|
||||
int ret = 0;
|
||||
|
||||
assert(txq_data);
|
||||
@ -517,7 +520,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
|
||||
ERROR("Failed to retrieve UAR info, invalid libmlx5.so version");
|
||||
goto error;
|
||||
}
|
||||
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt));
|
||||
LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next);
|
||||
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
|
||||
@ -534,8 +537,8 @@ error:
|
||||
/**
|
||||
* Get an Tx queue Verbs object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* Queue index in DPDK Rx queue array
|
||||
*
|
||||
@ -543,8 +546,9 @@ error:
|
||||
* The Verbs object if it exists.
|
||||
*/
|
||||
struct mlx5_txq_ibv *
|
||||
mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_ctrl *txq_ctrl;
|
||||
|
||||
if (idx >= priv->txqs_n)
|
||||
@ -554,7 +558,7 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
|
||||
if (txq_ctrl->ibv) {
|
||||
rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
|
||||
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)txq_ctrl->ibv,
|
||||
rte_atomic32_read(&txq_ctrl->ibv->refcnt));
|
||||
}
|
||||
@ -564,8 +568,6 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Release an Tx verbs queue object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param txq_ibv
|
||||
* Verbs Tx queue object.
|
||||
*
|
||||
@ -573,11 +575,10 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
|
||||
* 0 on success, errno on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused,
|
||||
struct mlx5_txq_ibv *txq_ibv)
|
||||
mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
|
||||
{
|
||||
assert(txq_ibv);
|
||||
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("Verbs Tx queue %p: refcnt %d",
|
||||
(void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt));
|
||||
if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
|
||||
claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
|
||||
@ -592,14 +593,11 @@ mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Return true if a single reference exists on the object.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param txq_ibv
|
||||
* Verbs Tx queue object.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused,
|
||||
struct mlx5_txq_ibv *txq_ibv)
|
||||
mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv)
|
||||
{
|
||||
assert(txq_ibv);
|
||||
return (rte_atomic32_read(&txq_ibv->refcnt) == 1);
|
||||
@ -608,20 +606,21 @@ mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused,
|
||||
/**
|
||||
* Verify the Verbs Tx queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_ibv_verify(struct priv *priv)
|
||||
mlx5_txq_ibv_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
int ret = 0;
|
||||
struct mlx5_txq_ibv *txq_ibv;
|
||||
|
||||
LIST_FOREACH(txq_ibv, &priv->txqsibv, next) {
|
||||
DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv,
|
||||
DEBUG("%p: Verbs Tx queue %p still referenced", (void *)dev,
|
||||
(void *)txq_ibv);
|
||||
++ret;
|
||||
}
|
||||
@ -645,7 +644,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
|
||||
unsigned int txq_inline;
|
||||
unsigned int txqs_inline;
|
||||
unsigned int inline_max_packet_sz;
|
||||
eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev);
|
||||
eth_tx_burst_t tx_pkt_burst =
|
||||
mlx5_select_tx_function(txq_ctrl->priv->dev);
|
||||
int is_empw_func = is_empw_burst_func(tx_pkt_burst);
|
||||
int tso = !!(txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_TCP_TSO);
|
||||
|
||||
@ -731,8 +731,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
|
||||
/**
|
||||
* Create a DPDK Tx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
* @param desc
|
||||
@ -746,10 +746,10 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
|
||||
* A DPDK queue object on success.
|
||||
*/
|
||||
struct mlx5_txq_ctrl *
|
||||
mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket,
|
||||
const struct rte_eth_txconf *conf)
|
||||
mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
|
||||
unsigned int socket, const struct rte_eth_txconf *conf)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_ctrl *tmpl;
|
||||
|
||||
tmpl = rte_calloc_socket("TXQ", 1,
|
||||
@ -773,7 +773,7 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
(struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1);
|
||||
tmpl->txq.stats.idx = idx;
|
||||
rte_atomic32_inc(&tmpl->refcnt);
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
|
||||
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
|
||||
return tmpl;
|
||||
@ -782,8 +782,8 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
/**
|
||||
* Get a Tx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -791,8 +791,9 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
|
||||
* A pointer to the queue if it exists.
|
||||
*/
|
||||
struct mlx5_txq_ctrl *
|
||||
mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
|
||||
mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_ctrl *ctrl = NULL;
|
||||
|
||||
if ((*priv->txqs)[idx]) {
|
||||
@ -800,15 +801,15 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
|
||||
txq);
|
||||
unsigned int i;
|
||||
|
||||
mlx5_priv_txq_ibv_get(priv, idx);
|
||||
mlx5_txq_ibv_get(dev, idx);
|
||||
for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
|
||||
if (ctrl->txq.mp2mr[i])
|
||||
claim_nonzero
|
||||
(priv_mr_get(priv,
|
||||
(mlx5_mr_get(dev,
|
||||
ctrl->txq.mp2mr[i]->mp));
|
||||
}
|
||||
rte_atomic32_inc(&ctrl->refcnt);
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)ctrl, rte_atomic32_read(&ctrl->refcnt));
|
||||
}
|
||||
return ctrl;
|
||||
@ -817,8 +818,8 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Release a Tx queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -826,8 +827,9 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
|
||||
* 0 on success, errno on failure.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
|
||||
mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
unsigned int i;
|
||||
struct mlx5_txq_ctrl *txq;
|
||||
size_t page_size = sysconf(_SC_PAGESIZE);
|
||||
@ -835,18 +837,18 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
|
||||
if (!(*priv->txqs)[idx])
|
||||
return 0;
|
||||
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
|
||||
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
|
||||
(void *)txq, rte_atomic32_read(&txq->refcnt));
|
||||
if (txq->ibv) {
|
||||
int ret;
|
||||
|
||||
ret = mlx5_priv_txq_ibv_release(priv, txq->ibv);
|
||||
ret = mlx5_txq_ibv_release(txq->ibv);
|
||||
if (!ret)
|
||||
txq->ibv = NULL;
|
||||
}
|
||||
for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
|
||||
if (txq->txq.mp2mr[i]) {
|
||||
priv_mr_release(priv, txq->txq.mp2mr[i]);
|
||||
mlx5_mr_release(txq->txq.mp2mr[i]);
|
||||
txq->txq.mp2mr[i] = NULL;
|
||||
}
|
||||
}
|
||||
@ -866,8 +868,8 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Verify if the queue can be released.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
* @param idx
|
||||
* TX queue index.
|
||||
*
|
||||
@ -875,8 +877,9 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
|
||||
* 1 if the queue can be released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx)
|
||||
mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_ctrl *txq;
|
||||
|
||||
if (!(*priv->txqs)[idx])
|
||||
@ -888,20 +891,21 @@ mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx)
|
||||
/**
|
||||
* Verify the Tx Queue list is empty
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param dev
|
||||
* Pointer to Ethernet device.
|
||||
*
|
||||
* @return
|
||||
* The number of object not released.
|
||||
*/
|
||||
int
|
||||
mlx5_priv_txq_verify(struct priv *priv)
|
||||
mlx5_txq_verify(struct rte_eth_dev *dev)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_txq_ctrl *txq;
|
||||
int ret = 0;
|
||||
|
||||
LIST_FOREACH(txq, &priv->txqsctrl, next) {
|
||||
DEBUG("%p: Tx Queue %p still referenced", (void *)priv,
|
||||
DEBUG("%p: Tx Queue %p still referenced", (void *)dev,
|
||||
(void *)txq);
|
||||
++ret;
|
||||
}
|
||||
|
@ -79,55 +79,11 @@ mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
|
||||
++priv->vlan_filter_n;
|
||||
}
|
||||
if (dev->data->dev_started)
|
||||
priv_dev_traffic_restart(priv, dev);
|
||||
mlx5_traffic_restart(dev);
|
||||
out:
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set/reset VLAN stripping for a specific queue.
|
||||
*
|
||||
* @param priv
|
||||
* Pointer to private structure.
|
||||
* @param idx
|
||||
* RX queue index.
|
||||
* @param on
|
||||
* Enable/disable VLAN stripping.
|
||||
*/
|
||||
static void
|
||||
priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on)
|
||||
{
|
||||
struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl =
|
||||
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
|
||||
struct ibv_wq_attr mod;
|
||||
uint16_t vlan_offloads =
|
||||
(on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
|
||||
0;
|
||||
int err;
|
||||
|
||||
DEBUG("set VLAN offloads 0x%x for port %d queue %d",
|
||||
vlan_offloads, rxq->port_id, idx);
|
||||
if (!rxq_ctrl->ibv) {
|
||||
/* Update related bits in RX queue. */
|
||||
rxq->vlan_strip = !!on;
|
||||
return;
|
||||
}
|
||||
mod = (struct ibv_wq_attr){
|
||||
.attr_mask = IBV_WQ_ATTR_FLAGS,
|
||||
.flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
|
||||
.flags = vlan_offloads,
|
||||
};
|
||||
err = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
|
||||
if (err) {
|
||||
ERROR("%p: failed to modified stripping mode: %s",
|
||||
(void *)priv, strerror(err));
|
||||
return;
|
||||
}
|
||||
/* Update related bits in RX queue. */
|
||||
rxq->vlan_strip = !!on;
|
||||
}
|
||||
|
||||
/**
|
||||
* Callback to set/reset VLAN stripping for a specific queue.
|
||||
*
|
||||
@ -142,6 +98,14 @@ void
|
||||
mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
|
||||
{
|
||||
struct priv *priv = dev->data->dev_private;
|
||||
struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue];
|
||||
struct mlx5_rxq_ctrl *rxq_ctrl =
|
||||
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
|
||||
struct ibv_wq_attr mod;
|
||||
uint16_t vlan_offloads =
|
||||
(on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
|
||||
0;
|
||||
int err;
|
||||
|
||||
/* Validate hw support */
|
||||
if (!priv->config.hw_vlan_strip) {
|
||||
@ -153,7 +117,26 @@ mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
|
||||
ERROR("VLAN stripping, invalid queue number %d", queue);
|
||||
return;
|
||||
}
|
||||
priv_vlan_strip_queue_set(priv, queue, on);
|
||||
DEBUG("set VLAN offloads 0x%x for port %d queue %d",
|
||||
vlan_offloads, rxq->port_id, queue);
|
||||
if (!rxq_ctrl->ibv) {
|
||||
/* Update related bits in RX queue. */
|
||||
rxq->vlan_strip = !!on;
|
||||
return;
|
||||
}
|
||||
mod = (struct ibv_wq_attr){
|
||||
.attr_mask = IBV_WQ_ATTR_FLAGS,
|
||||
.flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
|
||||
.flags = vlan_offloads,
|
||||
};
|
||||
err = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
|
||||
if (err) {
|
||||
ERROR("%p: failed to modified stripping mode: %s",
|
||||
(void *)dev, strerror(err));
|
||||
return;
|
||||
}
|
||||
/* Update related bits in RX queue. */
|
||||
rxq->vlan_strip = !!on;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -180,7 +163,7 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask)
|
||||
}
|
||||
/* Run on every RX queue and set/reset VLAN stripping. */
|
||||
for (i = 0; (i != priv->rxqs_n); i++)
|
||||
priv_vlan_strip_queue_set(priv, i, hw_vlan_strip);
|
||||
mlx5_vlan_strip_queue_set(dev, i, hw_vlan_strip);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
Loading…
x
Reference in New Issue
Block a user