d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net/mlx5: prefix all functions with mlx5

Browse Source 
This change removes the need to distinguish unlocked priv_*() functions
which are therefore renamed using a mlx5_*() prefix for consistency.

At the same time, all functions from mlx5 uses a pointer to the ETH device
instead of the one to the PMD private data.

Signed-off-by: Nelio Laranjeiro <nelio.laranjeiro@6wind.com>
Acked-by: Adrien Mazarguil <adrien.mazarguil@6wind.com>
This commit is contained in:
Nélio Laranjeiro 2018-03-05 13:21:04 +01:00 committed by Ferruh Yigit
parent 7b2423cd2e
commit af4f09f282
16 changed files with 877 additions and 1078 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
drivers/net/mlx5/mlx5.c
View File

@ -169,8 +169,8 @@ mlx5_dev_close(struct rte_eth_dev *dev)
(void *)dev,
((priv->ctx != NULL) ? priv->ctx->device->name : ""));
/* In case mlx5_dev_stop() has not been called. */
priv_dev_interrupt_handler_uninstall(priv, dev);
priv_dev_traffic_disable(priv, dev);
mlx5_dev_interrupt_handler_uninstall(dev);
mlx5_traffic_disable(dev);
/* Prevent crashes when queues are still in use. */
dev->rx_pkt_burst = removed_rx_burst;
dev->tx_pkt_burst = removed_tx_burst;
@ -178,7 +178,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
/* XXX race condition if mlx5_rx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->rxqs_n); ++i)
mlx5_priv_rxq_release(priv, i);
mlx5_rxq_release(dev, i);
priv->rxqs_n = 0;
priv->rxqs = NULL;
}
@ -186,7 +186,7 @@ mlx5_dev_close(struct rte_eth_dev *dev)
/* XXX race condition if mlx5_tx_burst() is still running. */
usleep(1000);
for (i = 0; (i != priv->txqs_n); ++i)
mlx5_priv_txq_release(priv, i);
mlx5_txq_release(dev, i);
priv->txqs_n = 0;
priv->txqs = NULL;
}
@ -201,31 +201,31 @@ mlx5_dev_close(struct rte_eth_dev *dev)
if (priv->reta_idx != NULL)
rte_free(priv->reta_idx);
if (priv->primary_socket)
priv_socket_uninit(priv);
ret = mlx5_priv_hrxq_ibv_verify(priv);
mlx5_socket_uninit(dev);
ret = mlx5_hrxq_ibv_verify(dev);
if (ret)
WARN("%p: some Hash Rx queue still remain", (void *)priv);
ret = mlx5_priv_ind_table_ibv_verify(priv);
WARN("%p: some Hash Rx queue still remain", (void *)dev);
ret = mlx5_ind_table_ibv_verify(dev);
if (ret)
WARN("%p: some Indirection table still remain", (void *)priv);
ret = mlx5_priv_rxq_ibv_verify(priv);
WARN("%p: some Indirection table still remain", (void *)dev);
ret = mlx5_rxq_ibv_verify(dev);
if (ret)
WARN("%p: some Verbs Rx queue still remain", (void *)priv);
ret = mlx5_priv_rxq_verify(priv);
WARN("%p: some Verbs Rx queue still remain", (void *)dev);
ret = mlx5_rxq_verify(dev);
if (ret)
WARN("%p: some Rx Queues still remain", (void *)priv);
ret = mlx5_priv_txq_ibv_verify(priv);
WARN("%p: some Rx Queues still remain", (void *)dev);
ret = mlx5_txq_ibv_verify(dev);
if (ret)
WARN("%p: some Verbs Tx queue still remain", (void *)priv);
ret = mlx5_priv_txq_verify(priv);
WARN("%p: some Verbs Tx queue still remain", (void *)dev);
ret = mlx5_txq_verify(dev);
if (ret)
WARN("%p: some Tx Queues still remain", (void *)priv);
ret = priv_flow_verify(priv);
WARN("%p: some Tx Queues still remain", (void *)dev);
ret = mlx5_flow_verify(dev);
if (ret)
WARN("%p: some flows still remain", (void *)priv);
ret = priv_mr_verify(priv);
WARN("%p: some flows still remain", (void *)dev);
ret = mlx5_mr_verify(dev);
if (ret)
WARN("%p: some Memory Region still remain", (void *)priv);
WARN("%p: some Memory Region still remain", (void *)dev);
memset(priv, 0, sizeof(*priv));
}
@ -466,15 +466,16 @@ static void *uar_base;
/**
* Reserve UAR address space for primary process.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_uar_init_primary(struct priv *priv)
mlx5_uar_init_primary(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
void *addr = (void *)0;
int i;
const struct rte_mem_config *mcfg;
@ -516,15 +517,16 @@ priv_uar_init_primary(struct priv *priv)
* Reserve UAR address space for secondary process, align with
* primary process.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_uar_init_secondary(struct priv *priv)
mlx5_uar_init_secondary(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
void *addr;
int ret;
@ -690,7 +692,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
struct ibv_port_attr port_attr;
struct ibv_pd *pd = NULL;
struct priv *priv = NULL;
struct rte_eth_dev *eth_dev;
struct rte_eth_dev *eth_dev = NULL;
struct ibv_device_attr_ex device_attr_ex;
struct ether_addr mac;
struct ibv_device_attr_ex device_attr;
@ -721,20 +723,19 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
}
eth_dev->device = &pci_dev->device;
eth_dev->dev_ops = &mlx5_dev_sec_ops;
priv = eth_dev->data->dev_private;
err = priv_uar_init_secondary(priv);
err = mlx5_uar_init_secondary(eth_dev);
if (err < 0) {
err = -err;
goto error;
}
/* Receive command fd from primary process */
err = priv_socket_connect(priv);
err = mlx5_socket_connect(eth_dev);
if (err < 0) {
err = -err;
goto error;
}
/* Remap UAR for Tx queues. */
err = priv_tx_uar_remap(priv, err);
err = mlx5_tx_uar_remap(eth_dev, err);
if (err)
goto error;
/*
@ -743,9 +744,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
* secondary process.
*/
eth_dev->rx_pkt_burst =
priv_select_rx_function(priv, eth_dev);
mlx5_select_rx_function(eth_dev);
eth_dev->tx_pkt_burst =
priv_select_tx_function(priv, eth_dev);
mlx5_select_tx_function(eth_dev);
continue;
}
DEBUG("using port %u (%08" PRIx32 ")", port, test);
@ -859,11 +860,23 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
WARN("Rx CQE compression isn't supported");
config.cqe_comp = 0;
}
err = priv_uar_init_primary(priv);
eth_dev = rte_eth_dev_allocate(name);
if (eth_dev == NULL) {
ERROR("can not allocate rte ethdev");
err = ENOMEM;
goto port_error;
}
eth_dev->data->dev_private = priv;
priv->dev = eth_dev;
eth_dev->data->mac_addrs = priv->mac;
eth_dev->device = &pci_dev->device;
rte_eth_copy_pci_info(eth_dev, pci_dev);
eth_dev->device->driver = &mlx5_driver.driver;
err = mlx5_uar_init_primary(eth_dev);
if (err)
goto port_error;
/* Configure the first MAC address by default. */
if (priv_get_mac(priv, &mac.addr_bytes)) {
if (mlx5_get_mac(eth_dev, &mac.addr_bytes)) {
ERROR("cannot get MAC address, is mlx5_en loaded?"
" (errno: %s)", strerror(errno));
err = ENODEV;
@ -878,7 +891,7 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
{
char ifname[IF_NAMESIZE];
if (priv_get_ifname(priv, &ifname) == 0)
if (mlx5_get_ifname(eth_dev, &ifname) == 0)
DEBUG("port %u ifname is \"%s\"",
priv->port, ifname);
else
@ -886,25 +899,13 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
}
#endif
/* Get actual MTU if possible. */
priv_get_mtu(priv, &priv->mtu);
mlx5_get_mtu(eth_dev, &priv->mtu);
DEBUG("port %u MTU is %u", priv->port, priv->mtu);
eth_dev = rte_eth_dev_allocate(name);
if (eth_dev == NULL) {
ERROR("can not allocate rte ethdev");
err = ENOMEM;
goto port_error;
}
eth_dev->data->dev_private = priv;
eth_dev->data->mac_addrs = priv->mac;
eth_dev->device = &pci_dev->device;
rte_eth_copy_pci_info(eth_dev, pci_dev);
eth_dev->device->driver = &mlx5_driver.driver;
/*
* Initialize burst functions to prevent crashes before link-up.
*/
eth_dev->rx_pkt_burst = removed_rx_burst;
eth_dev->tx_pkt_burst = removed_tx_burst;
priv->dev = eth_dev;
eth_dev->dev_ops = &mlx5_dev_ops;
/* Register MAC address. */
claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0));
@ -919,10 +920,9 @@ mlx5_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
mlx5_glue->dv_set_context_attr(ctx,
MLX5DV_CTX_ATTR_BUF_ALLOCATORS,
(void *)((uintptr_t)&alctr));
/* Bring Ethernet device up. */
DEBUG("forcing Ethernet interface up");
priv_set_flags(priv, ~IFF_UP, IFF_UP);
mlx5_set_flags(eth_dev, ~IFF_UP, IFF_UP);
/* Store device configuration on private structure. */
priv->config = config;
continue;

90
drivers/net/mlx5/mlx5.h
View File

@ -163,18 +163,16 @@ int mlx5_getenv_int(const char *);
/* mlx5_ethdev.c */
struct priv *mlx5_get_priv(struct rte_eth_dev *dev);
int mlx5_is_secondary(void);
int priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE]);
int priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr);
int priv_get_mtu(struct priv *priv, uint16_t *mtu);
int priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags);
int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
unsigned int flags);
int mlx5_dev_configure(struct rte_eth_dev *dev);
void mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info);
const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev);
int priv_link_update(struct priv *priv, int wait_to_complete);
int priv_force_link_status_change(struct priv *priv, int status);
int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete);
int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status);
int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev,
struct rte_eth_fc_conf *fc_conf);
@ -183,22 +181,18 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev,
int mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
struct rte_pci_addr *pci_addr);
void mlx5_dev_link_status_handler(void *arg);
void mlx5_dev_interrupt_handler(void *cb_arg);
void priv_dev_interrupt_handler_uninstall(struct priv *priv,
struct rte_eth_dev *dev);
void priv_dev_interrupt_handler_install(struct priv *priv,
struct rte_eth_dev *dev);
void mlx5_dev_interrupt_handler(void *arg);
void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev);
void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev);
int mlx5_set_link_down(struct rte_eth_dev *dev);
int mlx5_set_link_up(struct rte_eth_dev *dev);
eth_tx_burst_t priv_select_tx_function(struct priv *priv,
struct rte_eth_dev *dev);
eth_rx_burst_t priv_select_rx_function(struct priv *priv,
struct rte_eth_dev *dev);
int mlx5_is_removed(struct rte_eth_dev *dev);
eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
/* mlx5_mac.c */
int priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN]);
int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN]);
void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac,
uint32_t index, uint32_t vmdq);
@ -210,7 +204,7 @@ int mlx5_rss_hash_update(struct rte_eth_dev *dev,
struct rte_eth_rss_conf *rss_conf);
int mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
struct rte_eth_rss_conf *rss_conf);
int priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size);
int mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size);
int mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size);
@ -227,13 +221,13 @@ void mlx5_allmulticast_disable(struct rte_eth_dev *dev);
/* mlx5_stats.c */
void priv_xstats_init(struct priv *priv);
void mlx5_xstats_init(struct rte_eth_dev *dev);
int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
void mlx5_stats_reset(struct rte_eth_dev *dev);
int mlx5_xstats_get(struct rte_eth_dev *dev,
struct rte_eth_xstat *stats, unsigned int n);
int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
unsigned int n);
void mlx5_xstats_reset(struct rte_eth_dev *dev);
int mlx5_xstats_get_names(struct rte_eth_dev *dev,
int mlx5_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
struct rte_eth_xstat_name *xstats_names,
unsigned int n);
@ -247,9 +241,8 @@ int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask);
int mlx5_dev_start(struct rte_eth_dev *dev);
void mlx5_dev_stop(struct rte_eth_dev *dev);
int priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev);
int priv_dev_traffic_disable(struct priv *priv, struct rte_eth_dev *dev);
int priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev);
int mlx5_traffic_enable(struct rte_eth_dev *dev);
int mlx5_traffic_disable(struct rte_eth_dev *dev);
int mlx5_traffic_restart(struct rte_eth_dev *dev);
/* mlx5_flow.c */
@ -259,21 +252,6 @@ int mlx5_flow_validate(struct rte_eth_dev *dev,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error);
void priv_flow_flush(struct priv *priv, struct mlx5_flows *list);
int priv_flow_create_drop_queue(struct priv *priv);
void priv_flow_stop(struct priv *priv, struct mlx5_flows *list);
int priv_flow_start(struct priv *priv, struct mlx5_flows *list);
int priv_flow_verify(struct priv *priv);
int priv_flow_create_drop_queue(struct priv *priv);
void priv_flow_delete_drop_queue(struct priv *priv);
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
struct rte_flow_item_eth *eth_spec,
struct rte_flow_item_eth *eth_mask,
struct rte_flow_item_vlan *vlan_spec,
struct rte_flow_item_vlan *vlan_mask);
int mlx5_ctrl_flow(struct rte_eth_dev *dev,
struct rte_flow_item_eth *eth_spec,
struct rte_flow_item_eth *eth_mask);
struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
@ -281,6 +259,7 @@ struct rte_flow *mlx5_flow_create(struct rte_eth_dev *dev,
struct rte_flow_error *error);
int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
struct rte_flow_error *error);
void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list);
int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
enum rte_flow_action_type action, void *data,
@ -291,19 +270,32 @@ int mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
enum rte_filter_type filter_type,
enum rte_filter_op filter_op,
void *arg);
int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list);
void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list);
int mlx5_flow_verify(struct rte_eth_dev *dev);
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
struct rte_flow_item_eth *eth_spec,
struct rte_flow_item_eth *eth_mask,
struct rte_flow_item_vlan *vlan_spec,
struct rte_flow_item_vlan *vlan_mask);
int mlx5_ctrl_flow(struct rte_eth_dev *dev,
struct rte_flow_item_eth *eth_spec,
struct rte_flow_item_eth *eth_mask);
int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev);
void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev);
/* mlx5_socket.c */
int priv_socket_init(struct priv *priv);
int priv_socket_uninit(struct priv *priv);
void priv_socket_handle(struct priv *priv);
int priv_socket_connect(struct priv *priv);
int mlx5_socket_init(struct rte_eth_dev *priv);
int mlx5_socket_uninit(struct rte_eth_dev *priv);
void mlx5_socket_handle(struct rte_eth_dev *priv);
int mlx5_socket_connect(struct rte_eth_dev *priv);
/* mlx5_mr.c */
struct mlx5_mr *priv_mr_new(struct priv *priv, struct rte_mempool *mp);
struct mlx5_mr *priv_mr_get(struct priv *priv, struct rte_mempool *mp);
int priv_mr_release(struct priv *priv, struct mlx5_mr *mr);
int priv_mr_verify(struct priv *priv);
struct mlx5_mr *mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp);
struct mlx5_mr *mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp);
int mlx5_mr_release(struct mlx5_mr *mr);
int mlx5_mr_verify(struct rte_eth_dev *dev);
#endif /* RTE_PMD_MLX5_H_ */

288
drivers/net/mlx5/mlx5_ethdev.c
View File

@ -95,8 +95,8 @@ struct ethtool_link_settings {
/**
* Get interface name from private structure.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
* @param[out] ifname
* Interface name output buffer.
*
@ -104,8 +104,9 @@ struct ethtool_link_settings {
* 0 on success, -1 on failure and errno is set.
*/
int
priv_get_ifname(const struct priv *priv, char (*ifname)[IF_NAMESIZE])
mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE])
{
struct priv *priv = dev->data->dev_private;
DIR *dir;
struct dirent *dent;
unsigned int dev_type = 0;
@ -176,8 +177,8 @@ try_dev_id:
/**
* Perform ifreq ioctl() on associated Ethernet device.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
* @param req
* Request number to pass to ioctl().
* @param[out] ifr
@ -187,14 +188,14 @@ try_dev_id:
* 0 on success, -1 on failure and errno is set.
*/
int
priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr)
{
int sock = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);
int ret = -1;
if (sock == -1)
return ret;
if (priv_get_ifname(priv, &ifr->ifr_name) == 0)
if (mlx5_get_ifname(dev, &ifr->ifr_name) == 0)
ret = ioctl(sock, req, ifr);
close(sock);
return ret;
@ -203,8 +204,8 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
/**
* Get device MTU.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param[out] mtu
* MTU value output buffer.
*
@ -212,10 +213,10 @@ priv_ifreq(const struct priv *priv, int req, struct ifreq *ifr)
* 0 on success, -1 on failure and errno is set.
*/
int
priv_get_mtu(struct priv *priv, uint16_t *mtu)
mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu)
{
struct ifreq request;
int ret = priv_ifreq(priv, SIOCGIFMTU, &request);
int ret = mlx5_ifreq(dev, SIOCGIFMTU, &request);
if (ret)
return ret;
@ -226,8 +227,8 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu)
/**
* Set device MTU.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param mtu
* MTU value to set.
*
@ -235,18 +236,18 @@ priv_get_mtu(struct priv *priv, uint16_t *mtu)
* 0 on success, -1 on failure and errno is set.
*/
static int
priv_set_mtu(struct priv *priv, uint16_t mtu)
mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
{
struct ifreq request = { .ifr_mtu = mtu, };
return priv_ifreq(priv, SIOCSIFMTU, &request);
return mlx5_ifreq(dev, SIOCSIFMTU, &request);
}
/**
* Set device flags.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param keep
* Bitmask for flags that must remain untouched.
* @param flags
@ -256,16 +257,16 @@ priv_set_mtu(struct priv *priv, uint16_t mtu)
* 0 on success, -1 on failure and errno is set.
*/
int
priv_set_flags(struct priv *priv, unsigned int keep, unsigned int flags)
mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep, unsigned int flags)
{
struct ifreq request;
int ret = priv_ifreq(priv, SIOCGIFFLAGS, &request);
int ret = mlx5_ifreq(dev, SIOCGIFFLAGS, &request);
if (ret)
return ret;
request.ifr_flags &= keep;
request.ifr_flags |= flags & ~keep;
return priv_ifreq(priv, SIOCSIFFLAGS, &request);
return mlx5_ifreq(dev, SIOCSIFFLAGS, &request);
}
/**
@ -288,11 +289,11 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
unsigned int reta_idx_n;
const uint8_t use_app_rss_key =
!!dev->data->dev_conf.rx_adv_conf.rss_conf.rss_key;
uint64_t supp_tx_offloads = mlx5_priv_get_tx_port_offloads(priv);
uint64_t supp_tx_offloads = mlx5_get_tx_port_offloads(dev);
uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
uint64_t supp_rx_offloads =
(mlx5_priv_get_rx_port_offloads(priv) |
mlx5_priv_get_rx_queue_offloads(priv));
(mlx5_get_rx_port_offloads() |
mlx5_get_rx_queue_offloads(dev));
uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
if ((tx_offloads & supp_tx_offloads) != tx_offloads) {
@ -349,7 +350,7 @@ mlx5_dev_configure(struct rte_eth_dev *dev)
reta_idx_n = (1 << log2above((rxqs_n & (rxqs_n - 1)) ?
priv->config.ind_table_max_size :
rxqs_n));
if (priv_rss_reta_index_resize(priv, reta_idx_n))
if (mlx5_rss_reta_index_resize(dev, reta_idx_n))
return ENOMEM;
/* When the number of RX queues is not a power of two, the remaining
* table entries are padded with reused WQs and hashes are not spread
@ -395,12 +396,11 @@ mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
info->max_rx_queues = max;
info->max_tx_queues = max;
info->max_mac_addrs = RTE_DIM(priv->mac);
info->rx_queue_offload_capa =
mlx5_priv_get_rx_queue_offloads(priv);
info->rx_offload_capa = (mlx5_priv_get_rx_port_offloads(priv) |
info->rx_queue_offload_capa = mlx5_get_rx_queue_offloads(dev);
info->rx_offload_capa = (mlx5_get_rx_port_offloads() |
info->rx_queue_offload_capa);
info->tx_offload_capa = mlx5_priv_get_tx_port_offloads(priv);
if (priv_get_ifname(priv, &ifname) == 0)
info->tx_offload_capa = mlx5_get_tx_port_offloads(dev);
if (mlx5_get_ifname(dev, &ifname) == 0)
info->if_index = if_nametoindex(ifname);
info->reta_size = priv->reta_idx_n ?
priv->reta_idx_n : config->ind_table_max_size;
@ -465,7 +465,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev)
struct rte_eth_link dev_link;
int link_speed = 0;
if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) {
WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
return -1;
}
@ -473,7 +473,7 @@ mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev)
dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
(ifr.ifr_flags & IFF_RUNNING));
ifr.ifr_data = (void *)&edata;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
strerror(errno));
return -1;
@ -527,7 +527,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
struct rte_eth_link dev_link;
uint64_t sc;
if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
if (mlx5_ifreq(dev, SIOCGIFFLAGS, &ifr)) {
WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
return -1;
}
@ -535,7 +535,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
(ifr.ifr_flags & IFF_RUNNING));
ifr.ifr_data = (void *)&gcmd;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
strerror(errno));
return -1;
@ -549,7 +549,7 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
*ecmd = gcmd;
ifr.ifr_data = (void *)ecmd;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
strerror(errno));
return -1;
@ -608,90 +608,50 @@ mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev)
/**
* Enable receiving and transmitting traffic.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
static void
priv_link_start(struct priv *priv)
mlx5_link_start(struct rte_eth_dev *dev)
{
struct rte_eth_dev *dev = priv->dev;
struct priv *priv = dev->data->dev_private;
int err;
dev->tx_pkt_burst = priv_select_tx_function(priv, dev);
dev->rx_pkt_burst = priv_select_rx_function(priv, dev);
err = priv_dev_traffic_enable(priv, dev);
dev->tx_pkt_burst = mlx5_select_tx_function(dev);
dev->rx_pkt_burst = mlx5_select_rx_function(dev);
err = mlx5_traffic_enable(dev);
if (err)
ERROR("%p: error occurred while configuring control flows: %s",
(void *)priv, strerror(err));
err = priv_flow_start(priv, &priv->flows);
(void *)dev, strerror(err));
err = mlx5_flow_start(dev, &priv->flows);
if (err)
ERROR("%p: error occurred while configuring flows: %s",
(void *)priv, strerror(err));
(void *)dev, strerror(err));
}
/**
* Disable receiving and transmitting traffic.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
static void
priv_link_stop(struct priv *priv)
mlx5_link_stop(struct rte_eth_dev *dev)
{
struct rte_eth_dev *dev = priv->dev;
struct priv *priv = dev->data->dev_private;
priv_flow_stop(priv, &priv->flows);
priv_dev_traffic_disable(priv, dev);
mlx5_flow_stop(dev, &priv->flows);
mlx5_traffic_disable(dev);
dev->rx_pkt_burst = removed_rx_burst;
dev->tx_pkt_burst = removed_tx_burst;
}
/**
* Retrieve physical link information and update rx/tx_pkt_burst callbacks
* accordingly.
*
* @param priv
* Pointer to private structure.
* @param wait_to_complete
* Wait for request completion (ignored).
*/
int
priv_link_update(struct priv *priv, int wait_to_complete __rte_unused)
{
struct rte_eth_dev *dev = priv->dev;
struct utsname utsname;
int ver[3];
int ret;
struct rte_eth_link dev_link = dev->data->dev_link;
if (uname(&utsname) == -1 ||
sscanf(utsname.release, "%d.%d.%d",
&ver[0], &ver[1], &ver[2]) != 3 ||
KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0))
ret = mlx5_link_update_unlocked_gset(dev);
else
ret = mlx5_link_update_unlocked_gs(dev);
/* If lsc interrupt is disabled, should always be ready for traffic. */
if (!dev->data->dev_conf.intr_conf.lsc) {
priv_link_start(priv);
return ret;
}
/* Re-select burst callbacks only if link status has been changed. */
if (!ret && dev_link.link_status != dev->data->dev_link.link_status) {
if (dev->data->dev_link.link_status == ETH_LINK_UP)
priv_link_start(priv);
else
priv_link_stop(priv);
}
return ret;
}
/**
* Querying the link status till it changes to the desired state.
* Number of query attempts is bounded by MLX5_MAX_LINK_QUERY_ATTEMPTS.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param status
* Link desired status.
*
@ -699,13 +659,13 @@ priv_link_update(struct priv *priv, int wait_to_complete __rte_unused)
* 0 on success, negative errno value on failure.
*/
int
priv_force_link_status_change(struct priv *priv, int status)
mlx5_force_link_status_change(struct rte_eth_dev *dev, int status)
{
int try = 0;
while (try < MLX5_MAX_LINK_QUERY_ATTEMPTS) {
priv_link_update(priv, 0);
if (priv->dev->data->dev_link.link_status == status)
mlx5_link_update(dev, 0);
if (dev->data->dev_link.link_status == status)
return 0;
try++;
sleep(1);
@ -727,10 +687,30 @@ priv_force_link_status_change(struct priv *priv, int status)
int
mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
{
struct priv *priv = dev->data->dev_private;
struct utsname utsname;
int ver[3];
int ret;
struct rte_eth_link dev_link = dev->data->dev_link;
ret = priv_link_update(priv, wait_to_complete);
if (uname(&utsname) == -1 ||
sscanf(utsname.release, "%d.%d.%d",
&ver[0], &ver[1], &ver[2]) != 3 ||
KERNEL_VERSION(ver[0], ver[1], ver[2]) < KERNEL_VERSION(4, 9, 0))
ret = mlx5_link_update_unlocked_gset(dev);
else
ret = mlx5_link_update_unlocked_gs(dev);
/* If lsc interrupt is disabled, should always be ready for traffic. */
if (!dev->data->dev_conf.intr_conf.lsc) {
mlx5_link_start(dev);
return ret;
}
/* Re-select burst callbacks only if link status has been changed. */
if (!ret && dev_link.link_status != dev->data->dev_link.link_status) {
if (dev->data->dev_link.link_status == ETH_LINK_UP)
mlx5_link_start(dev);
else
mlx5_link_stop(dev);
}
return ret;
}
@ -752,14 +732,14 @@ mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
uint16_t kern_mtu;
int ret = 0;
ret = priv_get_mtu(priv, &kern_mtu);
ret = mlx5_get_mtu(dev, &kern_mtu);
if (ret)
goto out;
/* Set kernel interface MTU first. */
ret = priv_set_mtu(priv, mtu);
ret = mlx5_set_mtu(dev, mtu);
if (ret)
goto out;
ret = priv_get_mtu(priv, &kern_mtu);
ret = mlx5_get_mtu(dev, &kern_mtu);
if (ret)
goto out;
if (kern_mtu == mtu) {
@ -789,7 +769,6 @@ out:
int
mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
{
struct priv *priv = dev->data->dev_private;
struct ifreq ifr;
struct ethtool_pauseparam ethpause = {
.cmd = ETHTOOL_GPAUSEPARAM
@ -797,10 +776,9 @@ mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
int ret;
ifr.ifr_data = (void *)&ethpause;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
ret = errno;
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)"
" failed: %s",
WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM) failed: %s",
strerror(ret));
goto out;
}
@ -833,7 +811,6 @@ out:
int
mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
{
struct priv *priv = dev->data->dev_private;
struct ifreq ifr;
struct ethtool_pauseparam ethpause = {
.cmd = ETHTOOL_SPAUSEPARAM
@ -853,7 +830,7 @@ mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
ethpause.tx_pause = 1;
else
ethpause.tx_pause = 0;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr)) {
ret = errno;
WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
" failed: %s",
@ -919,18 +896,19 @@ mlx5_ibv_device_to_pci_addr(const struct ibv_device *device,
/**
* Update the link status.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* Zero if the callback process can be called immediately.
*/
static int
priv_link_status_update(struct priv *priv)
mlx5_link_status_update(struct rte_eth_dev *dev)
{
struct rte_eth_link *link = &priv->dev->data->dev_link;
struct priv *priv = dev->data->dev_private;
struct rte_eth_link *link = &dev->data->dev_link;
priv_link_update(priv, 0);
mlx5_link_update(dev, 0);
if (((link->link_speed == 0) && link->link_status) ||
((link->link_speed != 0) && !link->link_status)) {
/*
@ -955,8 +933,8 @@ priv_link_status_update(struct priv *priv)
/**
* Device status handler.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param events
* Pointer to event flags holder.
*
@ -964,8 +942,9 @@ priv_link_status_update(struct priv *priv)
* Events bitmap of callback process which can be called immediately.
*/
static uint32_t
priv_dev_status_handler(struct priv *priv)
mlx5_dev_status_handler(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct ibv_async_event event;
uint32_t ret = 0;
@ -975,10 +954,10 @@ priv_dev_status_handler(struct priv *priv)
break;
if ((event.event_type == IBV_EVENT_PORT_ACTIVE ||
event.event_type == IBV_EVENT_PORT_ERR) &&
(priv->dev->data->dev_conf.intr_conf.lsc == 1))
(dev->data->dev_conf.intr_conf.lsc == 1))
ret |= (1 << RTE_ETH_EVENT_INTR_LSC);
else if (event.event_type == IBV_EVENT_DEVICE_FATAL &&
priv->dev->data->dev_conf.intr_conf.rmv == 1)
dev->data->dev_conf.intr_conf.rmv == 1)
ret |= (1 << RTE_ETH_EVENT_INTR_RMV);
else
DEBUG("event type %d on port %d not handled",
@ -986,7 +965,7 @@ priv_dev_status_handler(struct priv *priv)
mlx5_glue->ack_async_event(&event);
}
if (ret & (1 << RTE_ETH_EVENT_INTR_LSC))
if (priv_link_status_update(priv))
if (mlx5_link_status_update(dev))
ret &= ~(1 << RTE_ETH_EVENT_INTR_LSC);
return ret;
}
@ -1005,7 +984,7 @@ mlx5_dev_link_status_handler(void *arg)
int ret;
priv->pending_alarm = 0;
ret = priv_link_status_update(priv);
ret = mlx5_link_status_update(dev);
if (!ret)
_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
}
@ -1022,10 +1001,9 @@ void
mlx5_dev_interrupt_handler(void *cb_arg)
{
struct rte_eth_dev *dev = cb_arg;
struct priv *priv = dev->data->dev_private;
uint32_t events;
events = priv_dev_status_handler(priv);
events = mlx5_dev_status_handler(dev);
if (events & (1 << RTE_ETH_EVENT_INTR_LSC))
_rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
if (events & (1 << RTE_ETH_EVENT_INTR_RMV))
@ -1042,22 +1020,21 @@ static void
mlx5_dev_handler_socket(void *cb_arg)
{
struct rte_eth_dev *dev = cb_arg;
struct priv *priv = dev->data->dev_private;
priv_socket_handle(priv);
mlx5_socket_handle(dev);
}
/**
* Uninstall interrupt handler.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to the rte_eth_dev structure.
* Pointer to Ethernet device.
*/
void
priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
if (dev->data->dev_conf.intr_conf.lsc ||
dev->data->dev_conf.intr_conf.rmv)
rte_intr_callback_unregister(&priv->intr_handle,
@ -1078,14 +1055,13 @@ priv_dev_interrupt_handler_uninstall(struct priv *priv, struct rte_eth_dev *dev)
/**
* Install interrupt handler.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to the rte_eth_dev structure.
* Pointer to Ethernet device.
*/
void
priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int rc, flags;
assert(priv->ctx->async_fd > 0);
@ -1103,7 +1079,7 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
rte_intr_callback_register(&priv->intr_handle,
mlx5_dev_interrupt_handler, dev);
}
rc = priv_socket_init(priv);
rc = mlx5_socket_init(dev);
if (!rc && priv->primary_socket) {
priv->intr_handle_socket.fd = priv->primary_socket;
priv->intr_handle_socket.type = RTE_INTR_HANDLE_EXT;
@ -1112,23 +1088,6 @@ priv_dev_interrupt_handler_install(struct priv *priv, struct rte_eth_dev *dev)
}
}
/**
* Change the link state (UP / DOWN).
*
* @param priv
* Pointer to private data structure.
* @param up
* Nonzero for link up, otherwise link down.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_dev_set_link(struct priv *priv, int up)
{
return priv_set_flags(priv, ~IFF_UP, up ? IFF_UP : ~IFF_UP);
}
/**
* DPDK callback to bring the link DOWN.
*
@ -1141,11 +1100,7 @@ priv_dev_set_link(struct priv *priv, int up)
int
mlx5_set_link_down(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int err;
err = priv_dev_set_link(priv, 0);
return err;
return mlx5_set_flags(dev, ~IFF_UP, ~IFF_UP);
}
/**
@ -1160,27 +1115,22 @@ mlx5_set_link_down(struct rte_eth_dev *dev)
int
mlx5_set_link_up(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int err;
err = priv_dev_set_link(priv, 1);
return err;
return mlx5_set_flags(dev, ~IFF_UP, IFF_UP);
}
/**
* Configure the TX function to use.
*
* @param priv
* Pointer to private data structure.
* @param dev
* Pointer to rte_eth_dev structure.
* Pointer to private data structure.
*
* @return
* Pointer to selected Tx burst function.
*/
eth_tx_burst_t
priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
mlx5_select_tx_function(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
eth_tx_burst_t tx_pkt_burst = mlx5_tx_burst;
struct mlx5_dev_config *config = &priv->config;
uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
@ -1194,8 +1144,8 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
if (vlan_insert || tso)
return tx_pkt_burst;
if (config->mps == MLX5_MPW_ENHANCED) {
if (priv_check_vec_tx_support(priv, dev) > 0) {
if (priv_check_raw_vec_tx_support(priv, dev) > 0)
if (mlx5_check_vec_tx_support(dev) > 0) {
if (mlx5_check_raw_vec_tx_support(dev) > 0)
tx_pkt_burst = mlx5_tx_burst_raw_vec;
else
tx_pkt_burst = mlx5_tx_burst_vec;
@ -1217,21 +1167,19 @@ priv_select_tx_function(struct priv *priv, struct rte_eth_dev *dev)
/**
* Configure the RX function to use.
*
* @param priv
* Pointer to private data structure.
* @param dev
* Pointer to rte_eth_dev structure.
* Pointer to private data structure.
*
* @return
* Pointer to selected Rx burst function.
*/
eth_rx_burst_t
priv_select_rx_function(struct priv *priv, __rte_unused struct rte_eth_dev *dev)
mlx5_select_rx_function(struct rte_eth_dev *dev)
{
eth_rx_burst_t rx_pkt_burst = mlx5_rx_burst;
assert(priv != NULL);
if (priv_check_vec_rx_support(priv) > 0) {
assert(dev != NULL);
if (mlx5_check_vec_rx_support(dev) > 0) {
rx_pkt_burst = mlx5_rx_burst_vec;
DEBUG("selected RX vectorized function");
}

342
drivers/net/mlx5/mlx5_flow.c
View File

@ -89,7 +89,7 @@ static int
mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id);
static int
mlx5_flow_create_count(struct priv *priv, struct mlx5_flow_parse *parser);
mlx5_flow_create_count(struct rte_eth_dev *dev, struct mlx5_flow_parse *parser);
/* Hash RX queue types. */
enum hash_rxq_type {
@ -515,8 +515,6 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
* Copy the RSS configuration from the user ones, of the rss_conf is null,
* uses the driver one.
*
* @param priv
* Pointer to private structure.
* @param parser
* Internal parser structure.
* @param rss_conf
@ -526,13 +524,12 @@ mlx5_flow_item_validate(const struct rte_flow_item *item,
* 0 on success, errno value on failure.
*/
static int
priv_flow_convert_rss_conf(struct priv *priv __rte_unused,
struct mlx5_flow_parse *parser,
mlx5_flow_convert_rss_conf(struct mlx5_flow_parse *parser,
const struct rte_eth_rss_conf *rss_conf)
{
/*
* This function is also called at the beginning of
* priv_flow_convert_actions() to initialize the parser with the
* mlx5_flow_convert_actions() to initialize the parser with the
* device default RSS configuration.
*/
if (rss_conf) {
@ -554,23 +551,17 @@ priv_flow_convert_rss_conf(struct priv *priv __rte_unused,
/**
* Extract attribute to the parser.
*
* @param priv
* Pointer to private structure.
* @param[in] attr
* Flow rule attributes.
* @param[out] error
* Perform verbose error reporting if not NULL.
* @param[in, out] parser
* Internal parser structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
priv_flow_convert_attributes(struct priv *priv __rte_unused,
const struct rte_flow_attr *attr,
struct rte_flow_error *error,
struct mlx5_flow_parse *parser __rte_unused)
mlx5_flow_convert_attributes(const struct rte_flow_attr *attr,
struct rte_flow_error *error)
{
if (attr->group) {
rte_flow_error_set(error, ENOTSUP,
@ -606,8 +597,8 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused,
/**
* Extract actions request to the parser.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param[in] actions
* Associated actions (list terminated by the END action).
* @param[out] error
@ -619,16 +610,18 @@ priv_flow_convert_attributes(struct priv *priv __rte_unused,
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
priv_flow_convert_actions(struct priv *priv,
mlx5_flow_convert_actions(struct rte_eth_dev *dev,
const struct rte_flow_action actions[],
struct rte_flow_error *error,
struct mlx5_flow_parse *parser)
{
struct priv *priv = dev->data->dev_private;
/*
* Add default RSS configuration necessary for Verbs to create QP even
* if no RSS is necessary.
*/
priv_flow_convert_rss_conf(priv, parser,
mlx5_flow_convert_rss_conf(parser,
(const struct rte_eth_rss_conf *)
&priv->rss_conf);
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; ++actions) {
@ -708,8 +701,7 @@ priv_flow_convert_actions(struct priv *priv,
for (n = 0; n < rss->num; ++n)
parser->queues[n] = rss->queue[n];
parser->queues_n = rss->num;
if (priv_flow_convert_rss_conf(priv, parser,
rss->rss_conf)) {
if (mlx5_flow_convert_rss_conf(parser, rss->rss_conf)) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
actions,
@ -763,8 +755,6 @@ exit_action_not_supported:
/**
* Validate items.
*
* @param priv
* Pointer to private structure.
* @param[in] items
* Pattern specification (list terminated by the END pattern item).
* @param[out] error
@ -776,8 +766,7 @@ exit_action_not_supported:
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
priv_flow_convert_items_validate(struct priv *priv __rte_unused,
const struct rte_flow_item items[],
mlx5_flow_convert_items_validate(const struct rte_flow_item items[],
struct rte_flow_error *error,
struct mlx5_flow_parse *parser)
{
@ -854,8 +843,6 @@ exit_item_not_supported:
/**
* Allocate memory space to store verbs flow attributes.
*
* @param priv
* Pointer to private structure.
* @param[in] priority
* Flow priority.
* @param[in] size
@ -867,8 +854,7 @@ exit_item_not_supported:
* A verbs flow attribute on success, NULL otherwise.
*/
static struct ibv_flow_attr *
priv_flow_convert_allocate(struct priv *priv __rte_unused,
unsigned int priority,
mlx5_flow_convert_allocate(unsigned int priority,
unsigned int size,
struct rte_flow_error *error)
{
@ -889,14 +875,11 @@ priv_flow_convert_allocate(struct priv *priv __rte_unused,
/**
* Finalise verbs flow attributes.
*
* @param priv
* Pointer to private structure.
* @param[in, out] parser
* Internal parser structure.
*/
static void
priv_flow_convert_finalise(struct priv *priv __rte_unused,
struct mlx5_flow_parse *parser)
mlx5_flow_convert_finalise(struct mlx5_flow_parse *parser)
{
const unsigned int ipv4 =
hash_rxq_init[parser->layer].ip_version == MLX5_IPV4;
@ -1015,8 +998,8 @@ fill:
/**
* Validate and convert a flow supported by the NIC.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param[in] attr
* Flow rule attributes.
* @param[in] pattern
@ -1032,7 +1015,7 @@ fill:
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
priv_flow_convert(struct priv *priv,
mlx5_flow_convert(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
@ -1049,16 +1032,16 @@ priv_flow_convert(struct priv *priv,
.layer = HASH_RXQ_ETH,
.mark_id = MLX5_FLOW_MARK_DEFAULT,
};
ret = priv_flow_convert_attributes(priv, attr, error, parser);
ret = mlx5_flow_convert_attributes(attr, error);
if (ret)
return ret;
ret = priv_flow_convert_actions(priv, actions, error, parser);
ret = mlx5_flow_convert_actions(dev, actions, error, parser);
if (ret)
return ret;
ret = priv_flow_convert_items_validate(priv, items, error, parser);
ret = mlx5_flow_convert_items_validate(items, error, parser);
if (ret)
return ret;
priv_flow_convert_finalise(priv, parser);
mlx5_flow_convert_finalise(parser);
/*
* Second step.
* Allocate the memory space to store verbs specifications.
@ -1070,8 +1053,7 @@ priv_flow_convert(struct priv *priv,
unsigned int offset = parser->queue[HASH_RXQ_ETH].offset;
parser->queue[HASH_RXQ_ETH].ibv_attr =
priv_flow_convert_allocate(priv, priority,
offset, error);
mlx5_flow_convert_allocate(priority, offset, error);
if (!parser->queue[HASH_RXQ_ETH].ibv_attr)
return ENOMEM;
parser->queue[HASH_RXQ_ETH].offset =
@ -1089,7 +1071,7 @@ priv_flow_convert(struct priv *priv,
continue;
offset = parser->queue[i].offset;
parser->queue[i].ibv_attr =
priv_flow_convert_allocate(priv, priority,
mlx5_flow_convert_allocate(priority,
offset, error);
if (!parser->queue[i].ibv_attr)
goto exit_enomem;
@ -1117,7 +1099,7 @@ priv_flow_convert(struct priv *priv,
if (parser->mark)
mlx5_flow_create_flag_mark(parser, parser->mark_id);
if (parser->count && parser->create) {
mlx5_flow_create_count(priv, parser);
mlx5_flow_create_count(dev, parser);
if (!parser->cs)
goto exit_count_error;
}
@ -1126,7 +1108,7 @@ priv_flow_convert(struct priv *priv,
* configuration.
*/
if (!parser->drop) {
priv_flow_convert_finalise(priv, parser);
mlx5_flow_convert_finalise(parser);
} else {
parser->queue[HASH_RXQ_ETH].ibv_attr->priority =
attr->priority +
@ -1578,8 +1560,8 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
/**
* Convert count action to Verbs specification.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param parser
* Pointer to MLX5 flow parser structure.
*
@ -1587,10 +1569,11 @@ mlx5_flow_create_flag_mark(struct mlx5_flow_parse *parser, uint32_t mark_id)
* 0 on success, errno value on failure.
*/
static int
mlx5_flow_create_count(struct priv *priv __rte_unused,
mlx5_flow_create_count(struct rte_eth_dev *dev __rte_unused,
struct mlx5_flow_parse *parser __rte_unused)
{
#ifdef HAVE_IBV_DEVICE_COUNTERS_SET_SUPPORT
struct priv *priv = dev->data->dev_private;
unsigned int size = sizeof(struct ibv_flow_spec_counter_action);
struct ibv_counter_set_init_attr init_attr = {0};
struct ibv_flow_spec_counter_action counter = {
@ -1612,8 +1595,8 @@ mlx5_flow_create_count(struct priv *priv __rte_unused,
/**
* Complete flow rule creation with a drop queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param parser
* Internal parser structure.
* @param flow
@ -1625,11 +1608,12 @@ mlx5_flow_create_count(struct priv *priv __rte_unused,
* 0 on success, errno value on failure.
*/
static int
priv_flow_create_action_queue_drop(struct priv *priv,
mlx5_flow_create_action_queue_drop(struct rte_eth_dev *dev,
struct mlx5_flow_parse *parser,
struct rte_flow *flow,
struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
struct ibv_flow_spec_action_drop *drop;
unsigned int size = sizeof(struct ibv_flow_spec_action_drop);
int err = 0;
@ -1684,8 +1668,8 @@ error:
/**
* Create hash Rx queues when RSS is enabled.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param parser
* Internal parser structure.
* @param flow
@ -1697,11 +1681,12 @@ error:
* 0 on success, a errno value otherwise and rte_errno is set.
*/
static int
priv_flow_create_action_queue_rss(struct priv *priv,
mlx5_flow_create_action_queue_rss(struct rte_eth_dev *dev,
struct mlx5_flow_parse *parser,
struct rte_flow *flow,
struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
for (i = 0; i != hash_rxq_init_n; ++i) {
@ -1715,21 +1700,21 @@ priv_flow_create_action_queue_rss(struct priv *priv,
if (!priv->dev->data->dev_started)
continue;
flow->frxq[i].hrxq =
mlx5_priv_hrxq_get(priv,
parser->rss_conf.rss_key,
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
parser->queues_n);
mlx5_hrxq_get(dev,
parser->rss_conf.rss_key,
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
parser->queues_n);
if (flow->frxq[i].hrxq)
continue;
flow->frxq[i].hrxq =
mlx5_priv_hrxq_new(priv,
parser->rss_conf.rss_key,
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
parser->queues_n);
mlx5_hrxq_new(dev,
parser->rss_conf.rss_key,
parser->rss_conf.rss_key_len,
hash_fields,
parser->queues,
parser->queues_n);
if (!flow->frxq[i].hrxq) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_HANDLE,
@ -1743,8 +1728,8 @@ priv_flow_create_action_queue_rss(struct priv *priv,
/**
* Complete flow rule creation.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param parser
* Internal parser structure.
* @param flow
@ -1756,11 +1741,12 @@ priv_flow_create_action_queue_rss(struct priv *priv,
* 0 on success, a errno value otherwise and rte_errno is set.
*/
static int
priv_flow_create_action_queue(struct priv *priv,
mlx5_flow_create_action_queue(struct rte_eth_dev *dev,
struct mlx5_flow_parse *parser,
struct rte_flow *flow,
struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
int err = 0;
unsigned int i;
unsigned int flows_n = 0;
@ -1768,7 +1754,7 @@ priv_flow_create_action_queue(struct priv *priv,
assert(priv->pd);
assert(priv->ctx);
assert(!parser->drop);
err = priv_flow_create_action_queue_rss(priv, parser, flow, error);
err = mlx5_flow_create_action_queue_rss(dev, parser, flow, error);
if (err)
goto error;
if (parser->count)
@ -1815,7 +1801,7 @@ error:
claim_zero(mlx5_glue->destroy_flow(ibv_flow));
}
if (flow->frxq[i].hrxq)
mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
if (flow->frxq[i].ibv_attr)
rte_free(flow->frxq[i].ibv_attr);
}
@ -1830,8 +1816,8 @@ error:
/**
* Convert a flow.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param list
* Pointer to a TAILQ flow list.
* @param[in] attr
@ -1847,19 +1833,19 @@ error:
* A flow on success, NULL otherwise.
*/
static struct rte_flow *
priv_flow_create(struct priv *priv,
struct mlx5_flows *list,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error)
mlx5_flow_list_create(struct rte_eth_dev *dev,
struct mlx5_flows *list,
const struct rte_flow_attr *attr,
const struct rte_flow_item items[],
const struct rte_flow_action actions[],
struct rte_flow_error *error)
{
struct mlx5_flow_parse parser = { .create = 1, };
struct rte_flow *flow = NULL;
unsigned int i;
int err;
err = priv_flow_convert(priv, attr, items, actions, error, &parser);
err = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
if (err)
goto exit;
flow = rte_calloc(__func__, 1,
@ -1883,10 +1869,10 @@ priv_flow_create(struct priv *priv,
memcpy(flow->rss_key, parser.rss_key, parser.rss_conf.rss_key_len);
/* finalise the flow. */
if (parser.drop)
err = priv_flow_create_action_queue_drop(priv, &parser, flow,
err = mlx5_flow_create_action_queue_drop(dev, &parser, flow,
error);
else
err = priv_flow_create_action_queue(priv, &parser, flow, error);
err = mlx5_flow_create_action_queue(dev, &parser, flow, error);
if (err)
goto exit;
TAILQ_INSERT_TAIL(list, flow, next);
@ -1915,11 +1901,10 @@ mlx5_flow_validate(struct rte_eth_dev *dev,
const struct rte_flow_action actions[],
struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
int ret;
struct mlx5_flow_parse parser = { .create = 0, };
ret = priv_flow_convert(priv, attr, items, actions, error, &parser);
ret = mlx5_flow_convert(dev, attr, items, actions, error, &parser);
return ret;
}
@ -1937,28 +1922,26 @@ mlx5_flow_create(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct priv *priv = dev->data->dev_private;
struct rte_flow *flow;
flow = priv_flow_create(priv, &priv->flows, attr, items, actions,
error);
return flow;
return mlx5_flow_list_create(dev, &priv->flows, attr, items, actions,
error);
}
/**
* Destroy a flow.
* Destroy a flow in a list.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param list
* Pointer to a TAILQ flow list.
* @param[in] flow
* Flow to destroy.
*/
static void
priv_flow_destroy(struct priv *priv,
struct mlx5_flows *list,
struct rte_flow *flow)
mlx5_flow_list_destroy(struct rte_eth_dev *dev, struct mlx5_flows *list,
struct rte_flow *flow)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
if (flow->drop || !flow->mark)
@ -2006,7 +1989,7 @@ free:
claim_zero(mlx5_glue->destroy_flow
(frxq->ibv_flow));
if (frxq->hrxq)
mlx5_priv_hrxq_release(priv, frxq->hrxq);
mlx5_hrxq_release(dev, frxq->hrxq);
if (frxq->ibv_attr)
rte_free(frxq->ibv_attr);
}
@ -2023,34 +2006,35 @@ free:
/**
* Destroy all flows.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param list
* Pointer to a TAILQ flow list.
*/
void
priv_flow_flush(struct priv *priv, struct mlx5_flows *list)
mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list)
{
while (!TAILQ_EMPTY(list)) {
struct rte_flow *flow;
flow = TAILQ_FIRST(list);
priv_flow_destroy(priv, list, flow);
mlx5_flow_list_destroy(dev, list, flow);
}
}
/**
* Create drop queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* 0 on success.
*/
int
priv_flow_create_drop_queue(struct priv *priv)
mlx5_flow_create_drop_queue(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_hrxq_drop *fdq = NULL;
assert(priv->pd);
@ -2131,12 +2115,13 @@ error:
/**
* Delete drop queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
void
priv_flow_delete_drop_queue(struct priv *priv)
mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_hrxq_drop *fdq = priv->flow_drop_queue;
if (!fdq)
@ -2156,14 +2141,15 @@ priv_flow_delete_drop_queue(struct priv *priv)
/**
* Remove all flows.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param list
* Pointer to a TAILQ flow list.
*/
void
priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list)
{
struct priv *priv = dev->data->dev_private;
struct rte_flow *flow;
TAILQ_FOREACH_REVERSE(flow, list, mlx5_flows, next) {
@ -2206,7 +2192,7 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
claim_zero(mlx5_glue->destroy_flow
(flow->frxq[i].ibv_flow));
flow->frxq[i].ibv_flow = NULL;
mlx5_priv_hrxq_release(priv, flow->frxq[i].hrxq);
mlx5_hrxq_release(dev, flow->frxq[i].hrxq);
flow->frxq[i].hrxq = NULL;
}
DEBUG("Flow %p removed", (void *)flow);
@ -2216,8 +2202,8 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
/**
* Add all flows.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param list
* Pointer to a TAILQ flow list.
*
@ -2225,8 +2211,9 @@ priv_flow_stop(struct priv *priv, struct mlx5_flows *list)
* 0 on success, a errno value otherwise and rte_errno is set.
*/
int
priv_flow_start(struct priv *priv, struct mlx5_flows *list)
mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list)
{
struct priv *priv = dev->data->dev_private;
struct rte_flow *flow;
TAILQ_FOREACH(flow, list, next) {
@ -2251,19 +2238,19 @@ priv_flow_start(struct priv *priv, struct mlx5_flows *list)
if (!flow->frxq[i].ibv_attr)
continue;
flow->frxq[i].hrxq =
mlx5_priv_hrxq_get(priv, flow->rss_conf.rss_key,
flow->rss_conf.rss_key_len,
hash_rxq_init[i].hash_fields,
(*flow->queues),
flow->queues_n);
mlx5_hrxq_get(dev, flow->rss_conf.rss_key,
flow->rss_conf.rss_key_len,
hash_rxq_init[i].hash_fields,
(*flow->queues),
flow->queues_n);
if (flow->frxq[i].hrxq)
goto flow_create;
flow->frxq[i].hrxq =
mlx5_priv_hrxq_new(priv, flow->rss_conf.rss_key,
flow->rss_conf.rss_key_len,
hash_rxq_init[i].hash_fields,
(*flow->queues),
flow->queues_n);
mlx5_hrxq_new(dev, flow->rss_conf.rss_key,
flow->rss_conf.rss_key_len,
hash_rxq_init[i].hash_fields,
(*flow->queues),
flow->queues_n);
if (!flow->frxq[i].hrxq) {
DEBUG("Flow %p cannot be applied",
(void *)flow);
@ -2293,19 +2280,20 @@ flow_create:
/**
* Verify the flow list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return the number of flows not released.
*/
int
priv_flow_verify(struct priv *priv)
mlx5_flow_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct rte_flow *flow;
int ret = 0;
TAILQ_FOREACH(flow, &priv->flows, next) {
DEBUG("%p: flow %p still referenced", (void *)priv,
DEBUG("%p: flow %p still referenced", (void *)dev,
(void *)flow);
++ret;
}
@ -2386,8 +2374,8 @@ mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
action_rss.local.rss_conf = &priv->rss_conf;
action_rss.local.num = priv->reta_idx_n;
actions[0].conf = (const void *)&action_rss.rss;
flow = priv_flow_create(priv, &priv->ctrl_flows, &attr, items, actions,
&error);
flow = mlx5_flow_list_create(dev, &priv->ctrl_flows, &attr, items,
actions, &error);
if (!flow)
return rte_errno;
return 0;
@ -2427,7 +2415,7 @@ mlx5_flow_destroy(struct rte_eth_dev *dev,
{
struct priv *priv = dev->data->dev_private;
priv_flow_destroy(priv, &priv->flows, flow);
mlx5_flow_list_destroy(dev, &priv->flows, flow);
return 0;
}
@ -2443,7 +2431,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
{
struct priv *priv = dev->data->dev_private;
priv_flow_flush(priv, &priv->flows);
mlx5_flow_list_flush(dev, &priv->flows);
return 0;
}
@ -2460,7 +2448,7 @@ mlx5_flow_flush(struct rte_eth_dev *dev,
* 0 on success, a errno value otherwise and rte_errno is set.
*/
static int
priv_flow_query_count(struct ibv_counter_set *cs,
mlx5_flow_query_count(struct ibv_counter_set *cs,
struct mlx5_flow_counter_stats *counter_stats,
struct rte_flow_query_count *query_count,
struct rte_flow_error *error)
@ -2510,7 +2498,7 @@ mlx5_flow_query(struct rte_eth_dev *dev __rte_unused,
int res = EINVAL;
if (flow->cs) {
res = priv_flow_query_count(flow->cs,
res = mlx5_flow_query_count(flow->cs,
&flow->counter_stats,
(struct rte_flow_query_count *)data,
error);
@ -2555,8 +2543,8 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
/**
* Convert a flow director filter to a generic flow.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
* @param fdir_filter
* Flow director filter to add.
* @param attributes
@ -2566,10 +2554,11 @@ mlx5_flow_isolate(struct rte_eth_dev *dev,
* 0 on success, errno value on error.
*/
static int
priv_fdir_filter_convert(struct priv *priv,
mlx5_fdir_filter_convert(struct rte_eth_dev *dev,
const struct rte_eth_fdir_filter *fdir_filter,
struct mlx5_fdir *attributes)
{
struct priv *priv = dev->data->dev_private;
const struct rte_eth_fdir_input *input = &fdir_filter->input;
/* Validate queue number. */
@ -2741,8 +2730,8 @@ priv_fdir_filter_convert(struct priv *priv,
/**
* Add new flow director filter and store it in list.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
* @param fdir_filter
* Flow director filter to add.
*
@ -2750,9 +2739,10 @@ priv_fdir_filter_convert(struct priv *priv,
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_add(struct priv *priv,
mlx5_fdir_filter_add(struct rte_eth_dev *dev,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_fdir attributes = {
.attr.group = 0,
.l2_mask = {
@ -2768,19 +2758,16 @@ priv_fdir_filter_add(struct priv *priv,
struct rte_flow *flow;
int ret;
ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
if (ret)
return -ret;
ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
attributes.actions, &error, &parser);
if (ret)
return -ret;
flow = priv_flow_create(priv,
&priv->flows,
&attributes.attr,
attributes.items,
attributes.actions,
&error);
flow = mlx5_flow_list_create(dev, &priv->flows, &attributes.attr,
attributes.items, attributes.actions,
&error);
if (flow) {
DEBUG("FDIR created %p", (void *)flow);
return 0;
@ -2791,8 +2778,8 @@ priv_fdir_filter_add(struct priv *priv,
/**
* Delete specific filter.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
* @param fdir_filter
* Filter to be deleted.
*
@ -2800,9 +2787,10 @@ priv_fdir_filter_add(struct priv *priv,
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_delete(struct priv *priv,
mlx5_fdir_filter_delete(struct rte_eth_dev *dev,
const struct rte_eth_fdir_filter *fdir_filter)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_fdir attributes = {
.attr.group = 0,
};
@ -2815,10 +2803,10 @@ priv_fdir_filter_delete(struct priv *priv,
unsigned int i;
int ret;
ret = priv_fdir_filter_convert(priv, fdir_filter, &attributes);
ret = mlx5_fdir_filter_convert(dev, fdir_filter, &attributes);
if (ret)
return -ret;
ret = priv_flow_convert(priv, &attributes.attr, attributes.items,
ret = mlx5_flow_convert(dev, &attributes.attr, attributes.items,
attributes.actions, &error, &parser);
if (ret)
goto exit;
@ -2876,7 +2864,7 @@ wrong_flow:
continue;
}
if (flow)
priv_flow_destroy(priv, &priv->flows, flow);
mlx5_flow_list_destroy(dev, &priv->flows, flow);
exit:
for (i = 0; i != hash_rxq_init_n; ++i) {
if (parser.queue[i].ibv_attr)
@ -2888,8 +2876,8 @@ exit:
/**
* Update queue for specific filter.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
* @param fdir_filter
* Filter to be updated.
*
@ -2897,41 +2885,44 @@ exit:
* 0 on success, errno value on failure.
*/
static int
priv_fdir_filter_update(struct priv *priv,
mlx5_fdir_filter_update(struct rte_eth_dev *dev,
const struct rte_eth_fdir_filter *fdir_filter)
{
int ret;
ret = priv_fdir_filter_delete(priv, fdir_filter);
ret = mlx5_fdir_filter_delete(dev, fdir_filter);
if (ret)
return ret;
ret = priv_fdir_filter_add(priv, fdir_filter);
ret = mlx5_fdir_filter_add(dev, fdir_filter);
return ret;
}
/**
* Flush all filters.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
*/
static void
priv_fdir_filter_flush(struct priv *priv)
mlx5_fdir_filter_flush(struct rte_eth_dev *dev)
{
priv_flow_flush(priv, &priv->flows);
struct priv *priv = dev->data->dev_private;
mlx5_flow_list_flush(dev, &priv->flows);
}
/**
* Get flow director information.
*
* @param priv
* Private structure.
* @param dev
* Pointer to Ethernet device.
* @param[out] fdir_info
* Resulting flow director information.
*/
static void
priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
mlx5_fdir_info_get(struct rte_eth_dev *dev, struct rte_eth_fdir_info *fdir_info)
{
struct priv *priv = dev->data->dev_private;
struct rte_eth_fdir_masks *mask =
&priv->dev->data->dev_conf.fdir_conf.mask;
@ -2949,8 +2940,8 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
/**
* Deal with flow director operations.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param filter_op
* Operation to perform.
* @param arg
@ -2960,8 +2951,10 @@ priv_fdir_info_get(struct priv *priv, struct rte_eth_fdir_info *fdir_info)
* 0 on success, errno value on failure.
*/
static int
priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
mlx5_fdir_ctrl_func(struct rte_eth_dev *dev, enum rte_filter_op filter_op,
void *arg)
{
struct priv *priv = dev->data->dev_private;
enum rte_fdir_mode fdir_mode =
priv->dev->data->dev_conf.fdir_conf.mode;
int ret = 0;
@ -2971,27 +2964,27 @@ priv_fdir_ctrl_func(struct priv *priv, enum rte_filter_op filter_op, void *arg)
if (fdir_mode != RTE_FDIR_MODE_PERFECT &&
fdir_mode != RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
ERROR("%p: flow director mode %d not supported",
(void *)priv, fdir_mode);
(void *)dev, fdir_mode);
return EINVAL;
}
switch (filter_op) {
case RTE_ETH_FILTER_ADD:
ret = priv_fdir_filter_add(priv, arg);
ret = mlx5_fdir_filter_add(dev, arg);
break;
case RTE_ETH_FILTER_UPDATE:
ret = priv_fdir_filter_update(priv, arg);
ret = mlx5_fdir_filter_update(dev, arg);
break;
case RTE_ETH_FILTER_DELETE:
ret = priv_fdir_filter_delete(priv, arg);
ret = mlx5_fdir_filter_delete(dev, arg);
break;
case RTE_ETH_FILTER_FLUSH:
priv_fdir_filter_flush(priv);
mlx5_fdir_filter_flush(dev);
break;
case RTE_ETH_FILTER_INFO:
priv_fdir_info_get(priv, arg);
mlx5_fdir_info_get(dev, arg);
break;
default:
DEBUG("%p: unknown operation %u", (void *)priv,
DEBUG("%p: unknown operation %u", (void *)dev,
filter_op);
ret = EINVAL;
break;
@ -3021,7 +3014,6 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
void *arg)
{
int ret = EINVAL;
struct priv *priv = dev->data->dev_private;
switch (filter_type) {
case RTE_ETH_FILTER_GENERIC:
@ -3030,7 +3022,7 @@ mlx5_dev_filter_ctrl(struct rte_eth_dev *dev,
*(const void **)arg = &mlx5_flow_ops;
return 0;
case RTE_ETH_FILTER_FDIR:
ret = priv_fdir_ctrl_func(priv, filter_op, arg);
ret = mlx5_fdir_ctrl_func(dev, filter_op, arg);
break;
default:
ERROR("%p: filter type (%d) not supported",

8
drivers/net/mlx5/mlx5_mac.c
View File

@ -35,8 +35,8 @@
/**
* Get MAC address by querying netdevice.
*
* @param[in] priv
* struct priv for the requested device.
* @param[in] dev
* Pointer to Ethernet device.
* @param[out] mac
* MAC address output buffer.
*
@ -44,11 +44,11 @@
* 0 on success, -1 on failure and errno is set.
*/
int
priv_get_mac(struct priv *priv, uint8_t (*mac)[ETHER_ADDR_LEN])
mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[ETHER_ADDR_LEN])
{
struct ifreq request;
if (priv_ifreq(priv, SIOCGIFHWADDR, &request))
if (mlx5_ifreq(dev, SIOCGIFHWADDR, &request))
return -1;
memcpy(mac, request.ifr_hwaddr.sa_data, ETHER_ADDR_LEN);
return 0;

86
drivers/net/mlx5/mlx5_mr.c
View File

@ -83,10 +83,6 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
* Register a Memory Region (MR) <-> Memory Pool (MP) association in
* txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
*
* This function should only be called by txq_mp2mr().
*
* @param priv
* Pointer to private structure.
* @param txq
* Pointer to TX queue structure.
* @param[in] mp
@ -98,29 +94,35 @@ mlx5_check_mempool(struct rte_mempool *mp, uintptr_t *start,
* mr on success, NULL on failure.
*/
struct mlx5_mr *
priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
struct rte_mempool *mp, unsigned int idx)
mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
unsigned int idx)
{
struct mlx5_txq_ctrl *txq_ctrl =
container_of(txq, struct mlx5_txq_ctrl, txq);
struct rte_eth_dev *dev;
struct mlx5_mr *mr;
rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
/* Add a new entry, register MR first. */
DEBUG("%p: discovered new memory pool \"%s\" (%p)",
(void *)txq_ctrl, mp->name, (void *)mp);
mr = priv_mr_get(priv, mp);
dev = txq_ctrl->priv->dev;
mr = mlx5_mr_get(dev, mp);
if (mr == NULL) {
if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
DEBUG("Using unregistered mempool 0x%p(%s) in secondary process,"
" please create mempool before rte_eth_dev_start()",
DEBUG("Using unregistered mempool 0x%p(%s) in "
"secondary process, please create mempool before "
" rte_eth_dev_start()",
(void *)mp, mp->name);
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
return NULL;
}
mr = priv_mr_new(priv, mp);
mr = mlx5_mr_new(dev, mp);
}
if (unlikely(mr == NULL)) {
DEBUG("%p: unable to configure MR, ibv_reg_mr() failed.",
(void *)txq_ctrl);
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
return NULL;
}
if (unlikely(idx == RTE_DIM(txq->mp2mr))) {
@ -128,7 +130,7 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
DEBUG("%p: MR <-> MP table full, dropping oldest entry.",
(void *)txq_ctrl);
--idx;
priv_mr_release(priv, txq->mp2mr[0]);
mlx5_mr_release(txq->mp2mr[0]);
memmove(&txq->mp2mr[0], &txq->mp2mr[1],
(sizeof(txq->mp2mr) - sizeof(txq->mp2mr[0])));
}
@ -137,35 +139,6 @@ priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
DEBUG("%p: new MR lkey for MP \"%s\" (%p): 0x%08" PRIu32,
(void *)txq_ctrl, mp->name, (void *)mp,
txq_ctrl->txq.mp2mr[idx]->lkey);
return mr;
}
/**
* Register a Memory Region (MR) <-> Memory Pool (MP) association in
* txq->mp2mr[]. If mp2mr[] is full, remove an entry first.
*
* This function should only be called by txq_mp2mr().
*
* @param txq
* Pointer to TX queue structure.
* @param[in] mp
* Memory Pool for which a Memory Region lkey must be returned.
* @param idx
* Index of the next available entry.
*
* @return
* mr on success, NULL on failure.
*/
struct mlx5_mr*
mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq, struct rte_mempool *mp,
unsigned int idx)
{
struct mlx5_txq_ctrl *txq_ctrl =
container_of(txq, struct mlx5_txq_ctrl, txq);
struct mlx5_mr *mr;
rte_spinlock_lock(&txq_ctrl->priv->mr_lock);
mr = priv_txq_mp2mr_reg(txq_ctrl->priv, txq, mp, idx);
rte_spinlock_unlock(&txq_ctrl->priv->mr_lock);
return mr;
}
@ -225,20 +198,20 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
if (rte_mempool_obj_iter(mp, txq_mp2mr_mbuf_check, &data) == 0 ||
data.ret == -1)
return;
mr = priv_mr_get(priv, mp);
mr = mlx5_mr_get(priv->dev, mp);
if (mr) {
priv_mr_release(priv, mr);
mlx5_mr_release(mr);
return;
}
priv_mr_new(priv, mp);
mlx5_mr_new(priv->dev, mp);
}
/**
* Register a new memory region from the mempool and store it in the memory
* region list.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param mp
* Pointer to the memory pool to register.
*
@ -246,8 +219,9 @@ mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg)
* The memory region on success.
*/
struct mlx5_mr *
priv_mr_new(struct priv *priv, struct rte_mempool *mp)
mlx5_mr_new(struct rte_eth_dev *dev, struct rte_mempool *mp)
{
struct priv *priv = dev->data->dev_private;
const struct rte_memseg *ms = rte_eal_get_physmem_layout();
uintptr_t start;
uintptr_t end;
@ -289,7 +263,7 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
mr->mp = mp;
mr->lkey = rte_cpu_to_be_32(mr->mr->lkey);
rte_atomic32_inc(&mr->refcnt);
DEBUG("%p: new Memory Region %p refcnt: %d", (void *)priv,
DEBUG("%p: new Memory Region %p refcnt: %d", (void *)dev,
(void *)mr, rte_atomic32_read(&mr->refcnt));
LIST_INSERT_HEAD(&priv->mr, mr, next);
return mr;
@ -298,8 +272,8 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
/**
* Search the memory region object in the memory region list.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param mp
* Pointer to the memory pool to register.
*
@ -307,8 +281,9 @@ priv_mr_new(struct priv *priv, struct rte_mempool *mp)
* The memory region on success.
*/
struct mlx5_mr *
priv_mr_get(struct priv *priv, struct rte_mempool *mp)
mlx5_mr_get(struct rte_eth_dev *dev, struct rte_mempool *mp)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_mr *mr;
assert(mp);
@ -335,7 +310,7 @@ priv_mr_get(struct priv *priv, struct rte_mempool *mp)
* 0 on success, errno on failure.
*/
int
priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr)
mlx5_mr_release(struct mlx5_mr *mr)
{
assert(mr);
DEBUG("Memory Region %p refcnt: %d",
@ -352,20 +327,21 @@ priv_mr_release(struct priv *priv __rte_unused, struct mlx5_mr *mr)
/**
* Verify the flow list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
priv_mr_verify(struct priv *priv)
mlx5_mr_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int ret = 0;
struct mlx5_mr *mr;
LIST_FOREACH(mr, &priv->mr, next) {
DEBUG("%p: mr %p still referenced", (void *)priv,
DEBUG("%p: mr %p still referenced", (void *)dev,
(void *)mr);
++ret;
}

109
drivers/net/mlx5/mlx5_rss.c
View File

@ -96,8 +96,8 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
/**
* Allocate/reallocate RETA index table.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @praram reta_size
* The size of the array to allocate.
*
@ -105,8 +105,9 @@ mlx5_rss_hash_conf_get(struct rte_eth_dev *dev,
* 0 on success, errno value on failure.
*/
int
priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size)
mlx5_rss_reta_index_resize(struct rte_eth_dev *dev, unsigned int reta_size)
{
struct priv *priv = dev->data->dev_private;
void *mem;
unsigned int old_size = priv->reta_idx_n;
@ -126,77 +127,6 @@ priv_rss_reta_index_resize(struct priv *priv, unsigned int reta_size)
return 0;
}
/**
* Query RETA table.
*
* @param priv
* Pointer to private structure.
* @param[in, out] reta_conf
* Pointer to the first RETA configuration structure.
* @param reta_size
* Number of entries.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_dev_rss_reta_query(struct priv *priv,
struct rte_eth_rss_reta_entry64 *reta_conf,
unsigned int reta_size)
{
unsigned int idx;
unsigned int i;
if (!reta_size || reta_size > priv->reta_idx_n)
return EINVAL;
/* Fill each entry of the table even if its bit is not set. */
for (idx = 0, i = 0; (i != reta_size); ++i) {
idx = i / RTE_RETA_GROUP_SIZE;
reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
(*priv->reta_idx)[i];
}
return 0;
}
/**
* Update RETA table.
*
* @param priv
* Pointer to private structure.
* @param[in] reta_conf
* Pointer to the first RETA configuration structure.
* @param reta_size
* Number of entries.
*
* @return
* 0 on success, errno value on failure.
*/
static int
priv_dev_rss_reta_update(struct priv *priv,
struct rte_eth_rss_reta_entry64 *reta_conf,
unsigned int reta_size)
{
unsigned int idx;
unsigned int i;
unsigned int pos;
int ret;
if (!reta_size)
return EINVAL;
ret = priv_rss_reta_index_resize(priv, reta_size);
if (ret)
return ret;
for (idx = 0, i = 0; (i != reta_size); ++i) {
idx = i / RTE_RETA_GROUP_SIZE;
pos = i % RTE_RETA_GROUP_SIZE;
if (((reta_conf[idx].mask >> i) & 0x1) == 0)
continue;
assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
}
return 0;
}
/**
* DPDK callback to get the RETA indirection table.
*
@ -215,11 +145,19 @@ mlx5_dev_rss_reta_query(struct rte_eth_dev *dev,
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size)
{
int ret;
struct priv *priv = dev->data->dev_private;
unsigned int idx;
unsigned int i;
ret = priv_dev_rss_reta_query(priv, reta_conf, reta_size);
return -ret;
if (!reta_size || reta_size > priv->reta_idx_n)
return -EINVAL;
/* Fill each entry of the table even if its bit is not set. */
for (idx = 0, i = 0; (i != reta_size); ++i) {
idx = i / RTE_RETA_GROUP_SIZE;
reta_conf[idx].reta[i % RTE_RETA_GROUP_SIZE] =
(*priv->reta_idx)[i];
}
return 0;
}
/**
@ -242,8 +180,23 @@ mlx5_dev_rss_reta_update(struct rte_eth_dev *dev,
{
int ret;
struct priv *priv = dev->data->dev_private;
unsigned int idx;
unsigned int i;
unsigned int pos;
ret = priv_dev_rss_reta_update(priv, reta_conf, reta_size);
if (!reta_size)
return -EINVAL;
ret = mlx5_rss_reta_index_resize(dev, reta_size);
if (ret)
return ret;
for (idx = 0, i = 0; (i != reta_size); ++i) {
idx = i / RTE_RETA_GROUP_SIZE;
pos = i % RTE_RETA_GROUP_SIZE;
if (((reta_conf[idx].mask >> i) & 0x1) == 0)
continue;
assert(reta_conf[idx].reta[pos] < priv->rxqs_n);
(*priv->reta_idx)[i] = reta_conf[idx].reta[pos];
}
if (dev->data->dev_started) {
mlx5_dev_stop(dev);
mlx5_dev_start(dev);

285
drivers/net/mlx5/mlx5_rxq.c
View File

@ -97,7 +97,7 @@ rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
(*rxq_ctrl->rxq.elts)[i] = buf;
}
/* If Rx vector is activated. */
if (rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
if (mlx5_rxq_check_vec_support(&rxq_ctrl->rxq) > 0) {
struct mlx5_rxq_data *rxq = &rxq_ctrl->rxq;
struct rte_mbuf *mbuf_init = &rxq->fake_mbuf;
int j;
@ -156,7 +156,7 @@ rxq_free_elts(struct mlx5_rxq_ctrl *rxq_ctrl)
* Some mbuf in the Ring belongs to the application. They cannot be
* freed.
*/
if (rxq_check_vec_support(rxq) > 0) {
if (mlx5_rxq_check_vec_support(rxq) > 0) {
for (i = 0; i < used; ++i)
(*rxq->elts)[(rxq->rq_ci + i) & q_mask] = NULL;
rxq->rq_pi = rxq->rq_ci;
@ -181,22 +181,23 @@ mlx5_rxq_cleanup(struct mlx5_rxq_ctrl *rxq_ctrl)
{
DEBUG("cleaning up %p", (void *)rxq_ctrl);
if (rxq_ctrl->ibv)
mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv);
mlx5_rxq_ibv_release(rxq_ctrl->ibv);
memset(rxq_ctrl, 0, sizeof(*rxq_ctrl));
}
/**
* Returns the per-queue supported offloads.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* Supported Rx offloads.
*/
uint64_t
mlx5_priv_get_rx_queue_offloads(struct priv *priv)
mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_dev_config *config = &priv->config;
uint64_t offloads = (DEV_RX_OFFLOAD_SCATTER |
DEV_RX_OFFLOAD_TIMESTAMP |
@ -217,13 +218,11 @@ mlx5_priv_get_rx_queue_offloads(struct priv *priv)
/**
* Returns the per-port supported offloads.
*
* @param priv
* Pointer to private structure.
* @return
* Supported Rx offloads.
*/
uint64_t
mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
mlx5_get_rx_port_offloads(void)
{
uint64_t offloads = DEV_RX_OFFLOAD_VLAN_FILTER;
@ -233,8 +232,8 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
/**
* Checks if the per-queue offload configuration is valid.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param offloads
* Per-queue offloads configuration.
*
@ -242,12 +241,11 @@ mlx5_priv_get_rx_port_offloads(struct priv *priv __rte_unused)
* 1 if the configuration is valid, 0 otherwise.
*/
static int
priv_is_rx_queue_offloads_allowed(struct priv *priv, uint64_t offloads)
mlx5_is_rx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads)
{
uint64_t port_offloads = priv->dev->data->dev_conf.rxmode.offloads;
uint64_t queue_supp_offloads =
mlx5_priv_get_rx_queue_offloads(priv);
uint64_t port_supp_offloads = mlx5_priv_get_rx_port_offloads(priv);
uint64_t port_offloads = dev->data->dev_conf.rxmode.offloads;
uint64_t queue_supp_offloads = mlx5_get_rx_queue_offloads(dev);
uint64_t port_supp_offloads = mlx5_get_rx_port_offloads();
if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
offloads)
@ -299,24 +297,24 @@ mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
(void *)dev, idx, priv->rxqs_n);
return -EOVERFLOW;
}
if (!priv_is_rx_queue_offloads_allowed(priv, conf->offloads)) {
if (!mlx5_is_rx_queue_offloads_allowed(dev, conf->offloads)) {
ret = ENOTSUP;
ERROR("%p: Rx queue offloads 0x%" PRIx64 " don't match port "
"offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64,
(void *)dev, conf->offloads,
dev->data->dev_conf.rxmode.offloads,
(mlx5_priv_get_rx_port_offloads(priv) |
mlx5_priv_get_rx_queue_offloads(priv)));
(mlx5_get_rx_port_offloads() |
mlx5_get_rx_queue_offloads(dev)));
goto out;
}
if (!mlx5_priv_rxq_releasable(priv, idx)) {
if (!mlx5_rxq_releasable(dev, idx)) {
ret = EBUSY;
ERROR("%p: unable to release queue index %u",
(void *)dev, idx);
goto out;
}
mlx5_priv_rxq_release(priv, idx);
rxq_ctrl = mlx5_priv_rxq_new(priv, idx, desc, socket, conf, mp);
mlx5_rxq_release(dev, idx);
rxq_ctrl = mlx5_rxq_new(dev, idx, desc, socket, conf, mp);
if (!rxq_ctrl) {
ERROR("%p: unable to allocate queue index %u",
(void *)dev, idx);
@ -347,24 +345,25 @@ mlx5_rx_queue_release(void *dpdk_rxq)
return;
rxq_ctrl = container_of(rxq, struct mlx5_rxq_ctrl, rxq);
priv = rxq_ctrl->priv;
if (!mlx5_priv_rxq_releasable(priv, rxq_ctrl->rxq.stats.idx))
if (!mlx5_rxq_releasable(priv->dev, rxq_ctrl->rxq.stats.idx))
rte_panic("Rx queue %p is still used by a flow and cannot be"
" removed\n", (void *)rxq_ctrl);
mlx5_priv_rxq_release(priv, rxq_ctrl->rxq.stats.idx);
mlx5_rxq_release(priv->dev, rxq_ctrl->rxq.stats.idx);
}
/**
* Allocate queue vector and fill epoll fd list for Rx interrupts.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, negative on failure.
*/
int
priv_rx_intr_vec_enable(struct priv *priv)
mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
unsigned int rxqs_n = priv->rxqs_n;
unsigned int n = RTE_MIN(rxqs_n, (uint32_t)RTE_MAX_RXTX_INTR_VEC_ID);
@ -373,7 +372,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
if (!priv->dev->data->dev_conf.intr_conf.rxq)
return 0;
priv_rx_intr_vec_disable(priv);
mlx5_rx_intr_vec_disable(dev);
intr_handle->intr_vec = malloc(n * sizeof(intr_handle->intr_vec[0]));
if (intr_handle->intr_vec == NULL) {
ERROR("failed to allocate memory for interrupt vector,"
@ -383,7 +382,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
intr_handle->type = RTE_INTR_HANDLE_EXT;
for (i = 0; i != n; ++i) {
/* This rxq ibv must not be released in this function. */
struct mlx5_rxq_ibv *rxq_ibv = mlx5_priv_rxq_ibv_get(priv, i);
struct mlx5_rxq_ibv *rxq_ibv = mlx5_rxq_ibv_get(dev, i);
int fd;
int flags;
int rc;
@ -400,7 +399,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
ERROR("too many Rx queues for interrupt vector size"
" (%d), Rx interrupts cannot be enabled",
RTE_MAX_RXTX_INTR_VEC_ID);
priv_rx_intr_vec_disable(priv);
mlx5_rx_intr_vec_disable(dev);
return -1;
}
fd = rxq_ibv->channel->fd;
@ -409,7 +408,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
if (rc < 0) {
ERROR("failed to make Rx interrupt file descriptor"
" %d non-blocking for queue index %d", fd, i);
priv_rx_intr_vec_disable(priv);
mlx5_rx_intr_vec_disable(dev);
return -1;
}
intr_handle->intr_vec[i] = RTE_INTR_VEC_RXTX_OFFSET + count;
@ -417,7 +416,7 @@ priv_rx_intr_vec_enable(struct priv *priv)
count++;
}
if (!count)
priv_rx_intr_vec_disable(priv);
mlx5_rx_intr_vec_disable(dev);
else
intr_handle->nb_efd = count;
return 0;
@ -426,12 +425,13 @@ priv_rx_intr_vec_enable(struct priv *priv)
/**
* Clean up Rx interrupts handler.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
void
priv_rx_intr_vec_disable(struct priv *priv)
mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct rte_intr_handle *intr_handle = priv->dev->intr_handle;
unsigned int i;
unsigned int rxqs_n = priv->rxqs_n;
@ -454,7 +454,7 @@ priv_rx_intr_vec_disable(struct priv *priv)
*/
rxq_data = (*priv->rxqs)[i];
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
mlx5_priv_rxq_ibv_release(priv, rxq_ctrl->ibv);
mlx5_rxq_ibv_release(rxq_ctrl->ibv);
}
free:
rte_intr_free_epoll_fd(intr_handle);
@ -516,13 +516,13 @@ mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
if (rxq_ctrl->irq) {
struct mlx5_rxq_ibv *rxq_ibv;
rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id);
rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
if (!rxq_ibv) {
ret = EINVAL;
goto exit;
}
mlx5_arm_cq(rxq_data, rxq_data->cq_arm_sn);
mlx5_priv_rxq_ibv_release(priv, rxq_ibv);
mlx5_rxq_ibv_release(rxq_ibv);
}
exit:
if (ret)
@ -560,7 +560,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
if (!rxq_ctrl->irq)
goto exit;
rxq_ibv = mlx5_priv_rxq_ibv_get(priv, rx_queue_id);
rxq_ibv = mlx5_rxq_ibv_get(dev, rx_queue_id);
if (!rxq_ibv) {
ret = EINVAL;
goto exit;
@ -574,7 +574,7 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
mlx5_glue->ack_cq_events(rxq_ibv->cq, 1);
exit:
if (rxq_ibv)
mlx5_priv_rxq_ibv_release(priv, rxq_ibv);
mlx5_rxq_ibv_release(rxq_ibv);
if (ret)
WARN("unable to disable interrupt on rx queue %d",
rx_queue_id);
@ -584,8 +584,8 @@ exit:
/**
* Create the Rx queue Verbs object.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* Queue index in DPDK Rx queue array
*
@ -593,8 +593,9 @@ exit:
* The Verbs object initialised if it can be created.
*/
struct mlx5_rxq_ibv *
mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
struct mlx5_rxq_ctrl *rxq_ctrl =
container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
@ -629,9 +630,9 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
}
tmpl->rxq_ctrl = rxq_ctrl;
/* Use the entire RX mempool as the memory region. */
tmpl->mr = priv_mr_get(priv, rxq_data->mp);
tmpl->mr = mlx5_mr_get(dev, rxq_data->mp);
if (!tmpl->mr) {
tmpl->mr = priv_mr_new(priv, rxq_data->mp);
tmpl->mr = mlx5_mr_new(dev, rxq_data->mp);
if (!tmpl->mr) {
ERROR("%p: MR creation failure", (void *)rxq_ctrl);
goto error;
@ -661,7 +662,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
* For vectorized Rx, it must not be doubled in order to
* make cq_ci and rq_ci aligned.
*/
if (rxq_check_vec_support(rxq_data) < 0)
if (mlx5_rxq_check_vec_support(rxq_data) < 0)
attr.cq.ibv.cqe *= 2;
} else if (config->cqe_comp && rxq_data->hw_timestamp) {
DEBUG("Rx CQE compression is disabled for HW timestamp");
@ -781,7 +782,7 @@ mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx)
*rxq_data->rq_db = rte_cpu_to_be_32(rxq_data->rq_ci);
DEBUG("%p: rxq updated with %p", (void *)rxq_ctrl, (void *)&tmpl);
rte_atomic32_inc(&tmpl->refcnt);
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev,
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
LIST_INSERT_HEAD(&priv->rxqsibv, tmpl, next);
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
@ -794,7 +795,7 @@ error:
if (tmpl->channel)
claim_zero(mlx5_glue->destroy_comp_channel(tmpl->channel));
if (tmpl->mr)
priv_mr_release(priv, tmpl->mr);
mlx5_mr_release(tmpl->mr);
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
return NULL;
}
@ -802,8 +803,8 @@ error:
/**
* Get an Rx queue Verbs object.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* Queue index in DPDK Rx queue array
*
@ -811,8 +812,9 @@ error:
* The Verbs object if it exists.
*/
struct mlx5_rxq_ibv *
mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_data *rxq_data = (*priv->rxqs)[idx];
struct mlx5_rxq_ctrl *rxq_ctrl;
@ -822,9 +824,9 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
return NULL;
rxq_ctrl = container_of(rxq_data, struct mlx5_rxq_ctrl, rxq);
if (rxq_ctrl->ibv) {
priv_mr_get(priv, rxq_data->mp);
mlx5_mr_get(dev, rxq_data->mp);
rte_atomic32_inc(&rxq_ctrl->ibv->refcnt);
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)dev,
(void *)rxq_ctrl->ibv,
rte_atomic32_read(&rxq_ctrl->ibv->refcnt));
}
@ -834,8 +836,6 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
/**
* Release an Rx verbs queue object.
*
* @param priv
* Pointer to private structure.
* @param rxq_ibv
* Verbs Rx queue object.
*
@ -843,7 +843,7 @@ mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx)
* 0 on success, errno value on failure.
*/
int
mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv)
{
int ret;
@ -851,10 +851,10 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
assert(rxq_ibv->wq);
assert(rxq_ibv->cq);
assert(rxq_ibv->mr);
ret = priv_mr_release(priv, rxq_ibv->mr);
ret = mlx5_mr_release(rxq_ibv->mr);
if (!ret)
rxq_ibv->mr = NULL;
DEBUG("%p: Verbs Rx queue %p: refcnt %d", (void *)priv,
DEBUG("Verbs Rx queue %p: refcnt %d",
(void *)rxq_ibv, rte_atomic32_read(&rxq_ibv->refcnt));
if (rte_atomic32_dec_and_test(&rxq_ibv->refcnt)) {
rxq_free_elts(rxq_ibv->rxq_ctrl);
@ -873,20 +873,21 @@ mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv)
/**
* Verify the Verbs Rx queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_rxq_ibv_verify(struct priv *priv)
mlx5_rxq_ibv_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int ret = 0;
struct mlx5_rxq_ibv *rxq_ibv;
LIST_FOREACH(rxq_ibv, &priv->rxqsibv, next) {
DEBUG("%p: Verbs Rx queue %p still referenced", (void *)priv,
DEBUG("%p: Verbs Rx queue %p still referenced", (void *)dev,
(void *)rxq_ibv);
++ret;
}
@ -896,14 +897,11 @@ mlx5_priv_rxq_ibv_verify(struct priv *priv)
/**
* Return true if a single reference exists on the object.
*
* @param priv
* Pointer to private structure.
* @param rxq_ibv
* Verbs Rx queue object.
*/
int
mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
struct mlx5_rxq_ibv *rxq_ibv)
mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv)
{
assert(rxq_ibv);
return (rte_atomic32_read(&rxq_ibv->refcnt) == 1);
@ -912,8 +910,8 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
/**
* Create a DPDK Rx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
* @param desc
@ -925,11 +923,11 @@ mlx5_priv_rxq_ibv_releasable(struct priv *priv __rte_unused,
* A DPDK queue object on success.
*/
struct mlx5_rxq_ctrl *
mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_rxconf *conf,
struct rte_mempool *mp)
mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_rxconf *conf,
struct rte_mempool *mp)
{
struct rte_eth_dev *dev = priv->dev;
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *tmpl;
unsigned int mb_len = rte_pktmbuf_data_room_size(mp);
struct mlx5_dev_config *config = &priv->config;
@ -1029,7 +1027,7 @@ mlx5_priv_rxq_new(struct priv *priv, uint16_t idx, uint16_t desc,
tmpl->rxq.elts =
(struct rte_mbuf *(*)[1 << tmpl->rxq.elts_n])(tmpl + 1);
rte_atomic32_inc(&tmpl->refcnt);
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
LIST_INSERT_HEAD(&priv->rxqsctrl, tmpl, next);
return tmpl;
@ -1041,8 +1039,8 @@ error:
/**
* Get a Rx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -1050,17 +1048,18 @@ error:
* A pointer to the queue if it exists.
*/
struct mlx5_rxq_ctrl *
mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *rxq_ctrl = NULL;
if ((*priv->rxqs)[idx]) {
rxq_ctrl = container_of((*priv->rxqs)[idx],
struct mlx5_rxq_ctrl,
rxq);
mlx5_priv_rxq_ibv_get(priv, idx);
mlx5_rxq_ibv_get(dev, idx);
rte_atomic32_inc(&rxq_ctrl->refcnt);
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
(void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt));
}
return rxq_ctrl;
@ -1069,8 +1068,8 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
/**
* Release a Rx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -1078,8 +1077,9 @@ mlx5_priv_rxq_get(struct priv *priv, uint16_t idx)
* 0 on success, errno value on failure.
*/
int
mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *rxq_ctrl;
if (!(*priv->rxqs)[idx])
@ -1089,11 +1089,11 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
if (rxq_ctrl->ibv) {
int ret;
ret = mlx5_priv_rxq_ibv_release(rxq_ctrl->priv, rxq_ctrl->ibv);
ret = mlx5_rxq_ibv_release(rxq_ctrl->ibv);
if (!ret)
rxq_ctrl->ibv = NULL;
}
DEBUG("%p: Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Rx queue %p: refcnt %d", (void *)dev,
(void *)rxq_ctrl, rte_atomic32_read(&rxq_ctrl->refcnt));
if (rte_atomic32_dec_and_test(&rxq_ctrl->refcnt)) {
LIST_REMOVE(rxq_ctrl, next);
@ -1107,8 +1107,8 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
/**
* Verify if the queue can be released.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -1116,8 +1116,9 @@ mlx5_priv_rxq_release(struct priv *priv, uint16_t idx)
* 1 if the queue can be released.
*/
int
mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx)
mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *rxq_ctrl;
if (!(*priv->rxqs)[idx])
@ -1129,20 +1130,21 @@ mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx)
/**
* Verify the Rx Queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_rxq_verify(struct priv *priv)
mlx5_rxq_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_ctrl *rxq_ctrl;
int ret = 0;
LIST_FOREACH(rxq_ctrl, &priv->rxqsctrl, next) {
DEBUG("%p: Rx Queue %p still referenced", (void *)priv,
DEBUG("%p: Rx Queue %p still referenced", (void *)dev,
(void *)rxq_ctrl);
++ret;
}
@ -1152,8 +1154,8 @@ mlx5_priv_rxq_verify(struct priv *priv)
/**
* Create an indirection table.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param queues
* Queues entering in the indirection table.
* @param queues_n
@ -1163,9 +1165,10 @@ mlx5_priv_rxq_verify(struct priv *priv)
* A new indirection table.
*/
struct mlx5_ind_table_ibv *
mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
uint16_t queues_n)
mlx5_ind_table_ibv_new(struct rte_eth_dev *dev, uint16_t queues[],
uint16_t queues_n)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_ind_table_ibv *ind_tbl;
const unsigned int wq_n = rte_is_power_of_2(queues_n) ?
log2above(queues_n) :
@ -1179,8 +1182,7 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
if (!ind_tbl)
return NULL;
for (i = 0; i != queues_n; ++i) {
struct mlx5_rxq_ctrl *rxq =
mlx5_priv_rxq_get(priv, queues[i]);
struct mlx5_rxq_ctrl *rxq = mlx5_rxq_get(dev, queues[i]);
if (!rxq)
goto error;
@ -1202,20 +1204,20 @@ mlx5_priv_ind_table_ibv_new(struct priv *priv, uint16_t queues[],
goto error;
rte_atomic32_inc(&ind_tbl->refcnt);
LIST_INSERT_HEAD(&priv->ind_tbls, ind_tbl, next);
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
return ind_tbl;
error:
rte_free(ind_tbl);
DEBUG("%p cannot create indirection table", (void *)priv);
DEBUG("%p cannot create indirection table", (void *)dev);
return NULL;
}
/**
* Get an indirection table.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param queues
* Queues entering in the indirection table.
* @param queues_n
@ -1225,9 +1227,10 @@ error:
* An indirection table if found.
*/
struct mlx5_ind_table_ibv *
mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
uint16_t queues_n)
mlx5_ind_table_ibv_get(struct rte_eth_dev *dev, uint16_t queues[],
uint16_t queues_n)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_ind_table_ibv *ind_tbl;
LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
@ -1241,10 +1244,10 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
unsigned int i;
rte_atomic32_inc(&ind_tbl->refcnt);
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
for (i = 0; i != ind_tbl->queues_n; ++i)
mlx5_priv_rxq_get(priv, ind_tbl->queues[i]);
mlx5_rxq_get(dev, ind_tbl->queues[i]);
}
return ind_tbl;
}
@ -1252,8 +1255,8 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
/**
* Release an indirection table.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param ind_table
* Indirection table to release.
*
@ -1261,18 +1264,18 @@ mlx5_priv_ind_table_ibv_get(struct priv *priv, uint16_t queues[],
* 0 on success, errno value on failure.
*/
int
mlx5_priv_ind_table_ibv_release(struct priv *priv,
struct mlx5_ind_table_ibv *ind_tbl)
mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
struct mlx5_ind_table_ibv *ind_tbl)
{
unsigned int i;
DEBUG("%p: Indirection table %p: refcnt %d", (void *)priv,
DEBUG("%p: Indirection table %p: refcnt %d", (void *)dev,
(void *)ind_tbl, rte_atomic32_read(&ind_tbl->refcnt));
if (rte_atomic32_dec_and_test(&ind_tbl->refcnt))
claim_zero(mlx5_glue->destroy_rwq_ind_table
(ind_tbl->ind_table));
for (i = 0; i != ind_tbl->queues_n; ++i)
claim_nonzero(mlx5_priv_rxq_release(priv, ind_tbl->queues[i]));
claim_nonzero(mlx5_rxq_release(dev, ind_tbl->queues[i]));
if (!rte_atomic32_read(&ind_tbl->refcnt)) {
LIST_REMOVE(ind_tbl, next);
rte_free(ind_tbl);
@ -1284,21 +1287,22 @@ mlx5_priv_ind_table_ibv_release(struct priv *priv,
/**
* Verify the Rx Queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_ind_table_ibv_verify(struct priv *priv)
mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_ind_table_ibv *ind_tbl;
int ret = 0;
LIST_FOREACH(ind_tbl, &priv->ind_tbls, next) {
DEBUG("%p: Verbs indirection table %p still referenced",
(void *)priv, (void *)ind_tbl);
(void *)dev, (void *)ind_tbl);
++ret;
}
return ret;
@ -1307,8 +1311,8 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv)
/**
* Create an Rx Hash queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param rss_key
* RSS key for the Rx hash queue.
* @param rss_key_len
@ -1325,17 +1329,18 @@ mlx5_priv_ind_table_ibv_verify(struct priv *priv)
* An hash Rx queue on success.
*/
struct mlx5_hrxq *
mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len,
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_hrxq *hrxq;
struct mlx5_ind_table_ibv *ind_tbl;
struct ibv_qp *qp;
queues_n = hash_fields ? queues_n : 1;
ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n);
ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
if (!ind_tbl)
ind_tbl = mlx5_priv_ind_table_ibv_new(priv, queues, queues_n);
ind_tbl = mlx5_ind_table_ibv_new(dev, queues, queues_n);
if (!ind_tbl)
return NULL;
qp = mlx5_glue->create_qp_ex
@ -1367,11 +1372,11 @@ mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
memcpy(hrxq->rss_key, rss_key, rss_key_len);
rte_atomic32_inc(&hrxq->refcnt);
LIST_INSERT_HEAD(&priv->hrxqs, hrxq, next);
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
return hrxq;
error:
mlx5_priv_ind_table_ibv_release(priv, ind_tbl);
mlx5_ind_table_ibv_release(dev, ind_tbl);
if (qp)
claim_zero(mlx5_glue->destroy_qp(qp));
return NULL;
@ -1380,8 +1385,8 @@ error:
/**
* Get an Rx Hash queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param rss_conf
* RSS configuration for the Rx hash queue.
* @param queues
@ -1394,9 +1399,10 @@ error:
* An hash Rx queue on success.
*/
struct mlx5_hrxq *
mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key, uint8_t rss_key_len,
uint64_t hash_fields, uint16_t queues[], uint16_t queues_n)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_hrxq *hrxq;
queues_n = hash_fields ? queues_n : 1;
@ -1409,15 +1415,15 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
continue;
if (hrxq->hash_fields != hash_fields)
continue;
ind_tbl = mlx5_priv_ind_table_ibv_get(priv, queues, queues_n);
ind_tbl = mlx5_ind_table_ibv_get(dev, queues, queues_n);
if (!ind_tbl)
continue;
if (ind_tbl != hrxq->ind_table) {
mlx5_priv_ind_table_ibv_release(priv, ind_tbl);
mlx5_ind_table_ibv_release(dev, ind_tbl);
continue;
}
rte_atomic32_inc(&hrxq->refcnt);
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
return hrxq;
}
@ -1427,8 +1433,8 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
/**
* Release the hash Rx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param hrxq
* Pointer to Hash Rx queue to release.
*
@ -1436,39 +1442,40 @@ mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key, uint8_t rss_key_len,
* 0 on success, errno value on failure.
*/
int
mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq)
mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hrxq)
{
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Hash Rx queue %p: refcnt %d", (void *)dev,
(void *)hrxq, rte_atomic32_read(&hrxq->refcnt));
if (rte_atomic32_dec_and_test(&hrxq->refcnt)) {
claim_zero(mlx5_glue->destroy_qp(hrxq->qp));
mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table);
mlx5_ind_table_ibv_release(dev, hrxq->ind_table);
LIST_REMOVE(hrxq, next);
rte_free(hrxq);
return 0;
}
claim_nonzero(mlx5_priv_ind_table_ibv_release(priv, hrxq->ind_table));
claim_nonzero(mlx5_ind_table_ibv_release(dev, hrxq->ind_table));
return EBUSY;
}
/**
* Verify the Rx Queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_hrxq_ibv_verify(struct priv *priv)
mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_hrxq *hrxq;
int ret = 0;
LIST_FOREACH(hrxq, &priv->hrxqs, next) {
DEBUG("%p: Verbs Hash Rx queue %p still referenced",
(void *)priv, (void *)hrxq);
(void *)dev, (void *)hrxq);
++ret;
}
return ret;

10
drivers/net/mlx5/mlx5_rxtx.c
View File

@ -1962,27 +1962,25 @@ mlx5_rx_burst_vec(void *dpdk_txq __rte_unused,
}
int __attribute__((weak))
priv_check_raw_vec_tx_support(struct priv *priv __rte_unused,
struct rte_eth_dev *dev __rte_unused)
mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
{
return -ENOTSUP;
}
int __attribute__((weak))
priv_check_vec_tx_support(struct priv *priv __rte_unused,
struct rte_eth_dev *dev __rte_unused)
mlx5_check_vec_tx_support(struct rte_eth_dev *dev __rte_unused)
{
return -ENOTSUP;
}
int __attribute__((weak))
rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq __rte_unused)
{
return -ENOTSUP;
}
int __attribute__((weak))
priv_check_vec_rx_support(struct priv *priv __rte_unused)
mlx5_check_vec_rx_support(struct rte_eth_dev *dev __rte_unused)
{
return -ENOTSUP;
}

115
drivers/net/mlx5/mlx5_rxtx.h
View File

@ -215,67 +215,64 @@ int mlx5_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_rxconf *conf,
struct rte_mempool *mp);
void mlx5_rx_queue_release(void *dpdk_rxq);
int priv_rx_intr_vec_enable(struct priv *priv);
void priv_rx_intr_vec_disable(struct priv *priv);
int mlx5_rx_intr_vec_enable(struct rte_eth_dev *dev);
void mlx5_rx_intr_vec_disable(struct rte_eth_dev *dev);
int mlx5_rx_intr_enable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
int mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id);
struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_new(struct priv *priv, uint16_t idx);
struct mlx5_rxq_ibv *mlx5_priv_rxq_ibv_get(struct priv *priv, uint16_t idx);
int mlx5_priv_rxq_ibv_release(struct priv *priv, struct mlx5_rxq_ibv *rxq_ibv);
int mlx5_priv_rxq_ibv_releasable(struct priv *priv,
struct mlx5_rxq_ibv *rxq_ibv);
int mlx5_priv_rxq_ibv_verify(struct priv *priv);
struct mlx5_rxq_ctrl *mlx5_priv_rxq_new(struct priv *priv, uint16_t idx,
uint16_t desc,
unsigned int socket,
const struct rte_eth_rxconf *conf,
struct rte_mempool *mp);
struct mlx5_rxq_ctrl *mlx5_priv_rxq_get(struct priv *priv, uint16_t idx);
int mlx5_priv_rxq_release(struct priv *priv, uint16_t idx);
int mlx5_priv_rxq_releasable(struct priv *priv, uint16_t idx);
int mlx5_priv_rxq_verify(struct priv *priv);
struct mlx5_rxq_ibv *mlx5_rxq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
struct mlx5_rxq_ibv *mlx5_rxq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_rxq_ibv_release(struct mlx5_rxq_ibv *rxq_ibv);
int mlx5_rxq_ibv_releasable(struct mlx5_rxq_ibv *rxq_ibv);
int mlx5_rxq_ibv_verify(struct rte_eth_dev *dev);
struct mlx5_rxq_ctrl *mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
uint16_t desc, unsigned int socket,
const struct rte_eth_rxconf *conf,
struct rte_mempool *mp);
struct mlx5_rxq_ctrl *mlx5_rxq_get(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_rxq_release(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_rxq_verify(struct rte_eth_dev *dev);
int rxq_alloc_elts(struct mlx5_rxq_ctrl *rxq_ctrl);
struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_new(struct priv *priv,
uint16_t queues[],
uint16_t queues_n);
struct mlx5_ind_table_ibv *mlx5_priv_ind_table_ibv_get(struct priv *priv,
uint16_t queues[],
uint16_t queues_n);
int mlx5_priv_ind_table_ibv_release(struct priv *priv,
struct mlx5_ind_table_ibv *ind_tbl);
int mlx5_priv_ind_table_ibv_verify(struct priv *priv);
struct mlx5_hrxq *mlx5_priv_hrxq_new(struct priv *priv, uint8_t *rss_key,
uint8_t rss_key_len, uint64_t hash_fields,
uint16_t queues[], uint16_t queues_n);
struct mlx5_hrxq *mlx5_priv_hrxq_get(struct priv *priv, uint8_t *rss_key,
uint8_t rss_key_len, uint64_t hash_fields,
uint16_t queues[], uint16_t queues_n);
int mlx5_priv_hrxq_release(struct priv *priv, struct mlx5_hrxq *hrxq);
int mlx5_priv_hrxq_ibv_verify(struct priv *priv);
uint64_t mlx5_priv_get_rx_port_offloads(struct priv *priv);
uint64_t mlx5_priv_get_rx_queue_offloads(struct priv *priv);
struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_new(struct rte_eth_dev *dev,
uint16_t queues[],
uint16_t queues_n);
struct mlx5_ind_table_ibv *mlx5_ind_table_ibv_get(struct rte_eth_dev *dev,
uint16_t queues[],
uint16_t queues_n);
int mlx5_ind_table_ibv_release(struct rte_eth_dev *dev,
struct mlx5_ind_table_ibv *ind_tbl);
int mlx5_ind_table_ibv_verify(struct rte_eth_dev *dev);
struct mlx5_hrxq *mlx5_hrxq_new(struct rte_eth_dev *dev, uint8_t *rss_key,
uint8_t rss_key_len, uint64_t hash_fields,
uint16_t queues[], uint16_t queues_n);
struct mlx5_hrxq *mlx5_hrxq_get(struct rte_eth_dev *dev, uint8_t *rss_key,
uint8_t rss_key_len, uint64_t hash_fields,
uint16_t queues[], uint16_t queues_n);
int mlx5_hrxq_release(struct rte_eth_dev *dev, struct mlx5_hrxq *hxrq);
int mlx5_hrxq_ibv_verify(struct rte_eth_dev *dev);
uint64_t mlx5_get_rx_port_offloads(void);
uint64_t mlx5_get_rx_queue_offloads(struct rte_eth_dev *dev);
/* mlx5_txq.c */
int mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_txconf *conf);
void mlx5_tx_queue_release(void *dpdk_txq);
int priv_tx_uar_remap(struct priv *priv, int fd);
struct mlx5_txq_ibv *mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx);
struct mlx5_txq_ibv *mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx);
int mlx5_priv_txq_ibv_release(struct priv *priv, struct mlx5_txq_ibv *txq_ibv);
int mlx5_priv_txq_ibv_releasable(struct priv *priv,
struct mlx5_txq_ibv *txq_ibv);
int mlx5_priv_txq_ibv_verify(struct priv *priv);
struct mlx5_txq_ctrl *mlx5_priv_txq_new(struct priv *priv, uint16_t idx,
uint16_t desc, unsigned int socket,
const struct rte_eth_txconf *conf);
struct mlx5_txq_ctrl *mlx5_priv_txq_get(struct priv *priv, uint16_t idx);
int mlx5_priv_txq_release(struct priv *priv, uint16_t idx);
int mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx);
int mlx5_priv_txq_verify(struct priv *priv);
int mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd);
struct mlx5_txq_ibv *mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx);
struct mlx5_txq_ibv *mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv);
int mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv);
int mlx5_txq_ibv_verify(struct rte_eth_dev *dev);
struct mlx5_txq_ctrl *mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx,
uint16_t desc, unsigned int socket,
const struct rte_eth_txconf *conf);
struct mlx5_txq_ctrl *mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx);
int mlx5_txq_verify(struct rte_eth_dev *dev);
void txq_alloc_elts(struct mlx5_txq_ctrl *txq_ctrl);
uint64_t mlx5_priv_get_tx_port_offloads(struct priv *priv);
uint64_t mlx5_get_tx_port_offloads(struct rte_eth_dev *dev);
/* mlx5_rxtx.c */
@ -299,26 +296,22 @@ int mlx5_rx_descriptor_status(void *rx_queue, uint16_t offset);
int mlx5_tx_descriptor_status(void *tx_queue, uint16_t offset);
/* Vectorized version of mlx5_rxtx.c */
int priv_check_raw_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev);
int priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev);
int rxq_check_vec_support(struct mlx5_rxq_data *rxq);
int priv_check_vec_rx_support(struct priv *priv);
int mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev);
int mlx5_check_vec_tx_support(struct rte_eth_dev *dev);
int mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq_data);
int mlx5_check_vec_rx_support(struct rte_eth_dev *dev);
uint16_t mlx5_tx_burst_raw_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
uint16_t mlx5_tx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
uint16_t mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts,
uint16_t mlx5_rx_burst_vec(void *dpdk_txq, struct rte_mbuf **pkts,
uint16_t pkts_n);
/* mlx5_mr.c */
void mlx5_mp2mr_iter(struct rte_mempool *mp, void *arg);
struct mlx5_mr *priv_txq_mp2mr_reg(struct priv *priv, struct mlx5_txq_data *txq,
struct rte_mempool *mp, unsigned int idx);
struct mlx5_mr *mlx5_txq_mp2mr_reg(struct mlx5_txq_data *txq,
struct rte_mempool *mp,
unsigned int idx);
struct rte_mempool *mp, unsigned int idx);
#ifndef NDEBUG
/**

25
drivers/net/mlx5/mlx5_rxtx_vec.c
View File

@ -223,17 +223,14 @@ mlx5_rx_burst_vec(void *dpdk_rxq, struct rte_mbuf **pkts, uint16_t pkts_n)
/**
* Check Tx queue flags are set for raw vectorized Tx.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to rte_eth_dev structure.
* Pointer to Ethernet device.
*
* @return
* 1 if supported, negative errno value if not.
*/
int __attribute__((cold))
priv_check_raw_vec_tx_support(__rte_unused struct priv *priv,
struct rte_eth_dev *dev)
mlx5_check_raw_vec_tx_support(struct rte_eth_dev *dev)
{
uint64_t offloads = dev->data->dev_conf.txmode.offloads;
@ -246,17 +243,16 @@ priv_check_raw_vec_tx_support(__rte_unused struct priv *priv,
/**
* Check a device can support vectorized TX.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to rte_eth_dev structure.
* Pointer to Ethernet device.
*
* @return
* 1 if supported, negative errno value if not.
*/
int __attribute__((cold))
priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev)
mlx5_check_vec_tx_support(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
uint64_t offloads = dev->data->dev_conf.txmode.offloads;
if (!priv->config.tx_vec_en ||
@ -277,7 +273,7 @@ priv_check_vec_tx_support(struct priv *priv, struct rte_eth_dev *dev)
* 1 if supported, negative errno value if not.
*/
int __attribute__((cold))
rxq_check_vec_support(struct mlx5_rxq_data *rxq)
mlx5_rxq_check_vec_support(struct mlx5_rxq_data *rxq)
{
struct mlx5_rxq_ctrl *ctrl =
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
@ -290,15 +286,16 @@ rxq_check_vec_support(struct mlx5_rxq_data *rxq)
/**
* Check a device can support vectorized RX.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* 1 if supported, negative errno value if not.
*/
int __attribute__((cold))
priv_check_vec_rx_support(struct priv *priv)
mlx5_check_vec_rx_support(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
uint16_t i;
if (!priv->config.rx_vec_en)
@ -309,7 +306,7 @@ priv_check_vec_rx_support(struct priv *priv)
if (!rxq)
continue;
if (rxq_check_vec_support(rxq) < 0)
if (mlx5_rxq_check_vec_support(rxq) < 0)
break;
}
if (i != priv->rxqs_n)

29
drivers/net/mlx5/mlx5_socket.c
View File

@ -18,15 +18,16 @@
/**
* Initialise the socket to communicate with the secondary process
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, errno value on failure.
*/
int
priv_socket_init(struct priv *priv)
mlx5_socket_init(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct sockaddr_un sun = {
.sun_family = AF_UNIX,
};
@ -79,15 +80,17 @@ out:
/**
* Un-Initialise the socket to communicate with the secondary process
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, errno value on failure.
*/
int
priv_socket_uninit(struct priv *priv)
mlx5_socket_uninit(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
MKSTR(path, "/var/tmp/%s_%d", MLX5_DRIVER_NAME, priv->primary_socket);
claim_zero(close(priv->primary_socket));
priv->primary_socket = 0;
@ -98,12 +101,13 @@ priv_socket_uninit(struct priv *priv)
/**
* Handle socket interrupts.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
void
priv_socket_handle(struct priv *priv)
mlx5_socket_handle(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int conn_sock;
int ret = 0;
struct cmsghdr *cmsg = NULL;
@ -179,15 +183,16 @@ out:
/**
* Connect to the primary process.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet structure.
*
* @return
* fd on success, negative errno value on failure.
*/
int
priv_socket_connect(struct priv *priv)
mlx5_socket_connect(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct sockaddr_un sun = {
.sun_family = AF_UNIX,
};

135
drivers/net/mlx5/mlx5_stats.c
View File

@ -122,8 +122,8 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
/**
* Read device counters table.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param[out] stats
* Counters table output buffer.
*
@ -131,8 +131,9 @@ static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init);
* 0 on success and stats is filled, negative on error.
*/
static int
priv_read_dev_counters(struct priv *priv, uint64_t *stats)
mlx5_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
unsigned int i;
struct ifreq ifr;
@ -143,7 +144,7 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats)
et_stats->cmd = ETHTOOL_GSTATS;
et_stats->n_stats = xstats_ctrl->stats_n;
ifr.ifr_data = (caddr_t)et_stats;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
WARN("unable to read statistic values from device");
return -1;
}
@ -173,20 +174,20 @@ priv_read_dev_counters(struct priv *priv, uint64_t *stats)
/**
* Query the number of statistics provided by ETHTOOL.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* Number of statistics on success, -1 on error.
*/
static int
priv_ethtool_get_stats_n(struct priv *priv) {
mlx5_ethtool_get_stats_n(struct rte_eth_dev *dev) {
struct ethtool_drvinfo drvinfo;
struct ifreq ifr;
drvinfo.cmd = ETHTOOL_GDRVINFO;
ifr.ifr_data = (caddr_t)&drvinfo;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
WARN("unable to query number of statistics");
return -1;
}
@ -196,12 +197,13 @@ priv_ethtool_get_stats_n(struct priv *priv) {
/**
* Init the structures to read device counters.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*/
void
priv_xstats_init(struct priv *priv)
mlx5_xstats_init(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
unsigned int i;
unsigned int j;
@ -210,7 +212,7 @@ priv_xstats_init(struct priv *priv)
unsigned int dev_stats_n;
unsigned int str_sz;
dev_stats_n = priv_ethtool_get_stats_n(priv);
dev_stats_n = mlx5_ethtool_get_stats_n(dev);
if (dev_stats_n < 1) {
WARN("no extended statistics available");
return;
@ -229,7 +231,7 @@ priv_xstats_init(struct priv *priv)
strings->string_set = ETH_SS_STATS;
strings->len = dev_stats_n;
ifr.ifr_data = (caddr_t)strings;
if (priv_ifreq(priv, SIOCETHTOOL, &ifr) != 0) {
if (mlx5_ifreq(dev, SIOCETHTOOL, &ifr) != 0) {
WARN("unable to get statistic names");
goto free;
}
@ -258,60 +260,54 @@ priv_xstats_init(struct priv *priv)
}
/* Copy to base at first time. */
assert(xstats_n <= MLX5_MAX_XSTATS);
priv_read_dev_counters(priv, xstats_ctrl->base);
mlx5_read_dev_counters(dev, xstats_ctrl->base);
free:
rte_free(strings);
}
/**
* Get device extended statistics.
* DPDK callback to get extended device statistics.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param[out] stats
* Pointer to rte extended stats table.
* @param n
* The size of the stats table.
*
* @return
* Number of extended stats on success and stats is filled,
* negative on error.
*/
static int
priv_xstats_get(struct priv *priv, struct rte_eth_xstat *stats)
int
mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
unsigned int n)
{
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
struct priv *priv = dev->data->dev_private;
unsigned int i;
unsigned int n = xstats_n;
uint64_t counters[n];
int ret = 0;
if (priv_read_dev_counters(priv, counters) < 0)
return -1;
for (i = 0; i != xstats_n; ++i) {
stats[i].id = i;
stats[i].value = (counters[i] - xstats_ctrl->base[i]);
if (n >= xstats_n && stats) {
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
int stats_n;
stats_n = mlx5_ethtool_get_stats_n(dev);
if (stats_n < 0)
return -1;
if (xstats_ctrl->stats_n != stats_n)
mlx5_xstats_init(dev);
ret = mlx5_read_dev_counters(dev, counters);
if (ret)
return ret;
for (i = 0; i != xstats_n; ++i) {
stats[i].id = i;
stats[i].value = (counters[i] - xstats_ctrl->base[i]);
}
}
return n;
}
/**
* Reset device extended statistics.
*
* @param priv
* Pointer to private structure.
*/
static void
priv_xstats_reset(struct priv *priv)
{
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
unsigned int i;
unsigned int n = xstats_n;
uint64_t counters[n];
if (priv_read_dev_counters(priv, counters) < 0)
return;
for (i = 0; i != n; ++i)
xstats_ctrl->base[i] = counters[i];
}
/**
* DPDK callback to get device statistics.
*
@ -408,41 +404,6 @@ mlx5_stats_reset(struct rte_eth_dev *dev)
#endif
}
/**
* DPDK callback to get extended device statistics.
*
* @param dev
* Pointer to Ethernet device structure.
* @param[out] stats
* Stats table output buffer.
* @param n
* The size of the stats table.
*
* @return
* Number of xstats on success, negative on failure.
*/
int
mlx5_xstats_get(struct rte_eth_dev *dev,
struct rte_eth_xstat *stats, unsigned int n)
{
struct priv *priv = dev->data->dev_private;
int ret = xstats_n;
if (n >= xstats_n && stats) {
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
int stats_n;
stats_n = priv_ethtool_get_stats_n(priv);
if (stats_n < 0) {
return -1;
}
if (xstats_ctrl->stats_n != stats_n)
priv_xstats_init(priv);
ret = priv_xstats_get(priv, stats);
}
return ret;
}
/**
* DPDK callback to clear device extended statistics.
*
@ -455,13 +416,19 @@ mlx5_xstats_reset(struct rte_eth_dev *dev)
struct priv *priv = dev->data->dev_private;
struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl;
int stats_n;
unsigned int i;
unsigned int n = xstats_n;
uint64_t counters[n];
stats_n = priv_ethtool_get_stats_n(priv);
stats_n = mlx5_ethtool_get_stats_n(dev);
if (stats_n < 0)
return;
if (xstats_ctrl->stats_n != stats_n)
priv_xstats_init(priv);
priv_xstats_reset(priv);
mlx5_xstats_init(dev);
if (mlx5_read_dev_counters(dev, counters) < 0)
return;
for (i = 0; i != n; ++i)
xstats_ctrl->base[i] = counters[i];
}
/**

118
drivers/net/mlx5/mlx5_trigger.c
View File

@ -21,12 +21,13 @@
* Pointer to Ethernet device structure.
*/
static void
priv_txq_stop(struct priv *priv)
mlx5_txq_stop(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
for (i = 0; i != priv->txqs_n; ++i)
mlx5_priv_txq_release(priv, i);
mlx5_txq_release(dev, i);
}
/**
@ -39,8 +40,9 @@ priv_txq_stop(struct priv *priv)
* 0 on success, errno on error.
*/
static int
priv_txq_start(struct priv *priv)
mlx5_txq_start(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
int ret = 0;
@ -48,28 +50,28 @@ priv_txq_start(struct priv *priv)
for (i = 0; i != priv->txqs_n; ++i) {
unsigned int idx = 0;
struct mlx5_mr *mr;
struct mlx5_txq_ctrl *txq_ctrl = mlx5_priv_txq_get(priv, i);
struct mlx5_txq_ctrl *txq_ctrl = mlx5_txq_get(dev, i);
if (!txq_ctrl)
continue;
LIST_FOREACH(mr, &priv->mr, next) {
priv_txq_mp2mr_reg(priv, &txq_ctrl->txq, mr->mp, idx++);
mlx5_txq_mp2mr_reg(&txq_ctrl->txq, mr->mp, idx++);
if (idx == MLX5_PMD_TX_MP_CACHE)
break;
}
txq_alloc_elts(txq_ctrl);
txq_ctrl->ibv = mlx5_priv_txq_ibv_new(priv, i);
txq_ctrl->ibv = mlx5_txq_ibv_new(dev, i);
if (!txq_ctrl->ibv) {
ret = ENOMEM;
goto error;
}
}
ret = priv_tx_uar_remap(priv, priv->ctx->cmd_fd);
ret = mlx5_tx_uar_remap(dev, priv->ctx->cmd_fd);
if (ret)
goto error;
return ret;
error:
priv_txq_stop(priv);
mlx5_txq_stop(dev);
return ret;
}
@ -80,12 +82,13 @@ error:
* Pointer to Ethernet device structure.
*/
static void
priv_rxq_stop(struct priv *priv)
mlx5_rxq_stop(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
for (i = 0; i != priv->rxqs_n; ++i)
mlx5_priv_rxq_release(priv, i);
mlx5_rxq_release(dev, i);
}
/**
@ -98,20 +101,21 @@ priv_rxq_stop(struct priv *priv)
* 0 on success, errno on error.
*/
static int
priv_rxq_start(struct priv *priv)
mlx5_rxq_start(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
int ret = 0;
for (i = 0; i != priv->rxqs_n; ++i) {
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_priv_rxq_get(priv, i);
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_get(dev, i);
if (!rxq_ctrl)
continue;
ret = rxq_alloc_elts(rxq_ctrl);
if (ret)
goto error;
rxq_ctrl->ibv = mlx5_priv_rxq_ibv_new(priv, i);
rxq_ctrl->ibv = mlx5_rxq_ibv_new(dev, i);
if (!rxq_ctrl->ibv) {
ret = ENOMEM;
goto error;
@ -119,7 +123,7 @@ priv_rxq_start(struct priv *priv)
}
return -ret;
error:
priv_rxq_stop(priv);
mlx5_rxq_stop(dev);
return -ret;
}
@ -142,7 +146,7 @@ mlx5_dev_start(struct rte_eth_dev *dev)
int err;
dev->data->dev_started = 1;
err = priv_flow_create_drop_queue(priv);
err = mlx5_flow_create_drop_queue(dev);
if (err) {
ERROR("%p: Drop queue allocation failed: %s",
(void *)dev, strerror(err));
@ -150,46 +154,46 @@ mlx5_dev_start(struct rte_eth_dev *dev)
}
DEBUG("%p: allocating and configuring hash RX queues", (void *)dev);
rte_mempool_walk(mlx5_mp2mr_iter, priv);
err = priv_txq_start(priv);
err = mlx5_txq_start(dev);
if (err) {
ERROR("%p: TXQ allocation failed: %s",
(void *)dev, strerror(err));
goto error;
}
err = priv_rxq_start(priv);
err = mlx5_rxq_start(dev);
if (err) {
ERROR("%p: RXQ allocation failed: %s",
(void *)dev, strerror(err));
goto error;
}
err = priv_rx_intr_vec_enable(priv);
err = mlx5_rx_intr_vec_enable(dev);
if (err) {
ERROR("%p: RX interrupt vector creation failed",
(void *)priv);
goto error;
}
priv_xstats_init(priv);
mlx5_xstats_init(dev);
/* Update link status and Tx/Rx callbacks for the first time. */
memset(&dev->data->dev_link, 0, sizeof(struct rte_eth_link));
INFO("Forcing port %u link to be up", dev->data->port_id);
err = priv_force_link_status_change(priv, ETH_LINK_UP);
err = mlx5_force_link_status_change(dev, ETH_LINK_UP);
if (err) {
DEBUG("Failed to set port %u link to be up",
dev->data->port_id);
goto error;
}
priv_dev_interrupt_handler_install(priv, dev);
mlx5_dev_interrupt_handler_install(dev);
return 0;
error:
/* Rollback. */
dev->data->dev_started = 0;
for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
priv_mr_release(priv, mr);
priv_flow_stop(priv, &priv->flows);
priv_dev_traffic_disable(priv, dev);
priv_txq_stop(priv);
priv_rxq_stop(priv);
priv_flow_delete_drop_queue(priv);
mlx5_mr_release(mr);
mlx5_flow_stop(dev, &priv->flows);
mlx5_traffic_disable(dev);
mlx5_txq_stop(dev);
mlx5_rxq_stop(dev);
mlx5_flow_delete_drop_queue(dev);
return err;
}
@ -214,21 +218,21 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
rte_wmb();
usleep(1000 * priv->rxqs_n);
DEBUG("%p: cleaning up and destroying hash RX queues", (void *)dev);
priv_flow_stop(priv, &priv->flows);
priv_dev_traffic_disable(priv, dev);
priv_rx_intr_vec_disable(priv);
priv_dev_interrupt_handler_uninstall(priv, dev);
priv_txq_stop(priv);
priv_rxq_stop(priv);
mlx5_flow_stop(dev, &priv->flows);
mlx5_traffic_disable(dev);
mlx5_rx_intr_vec_disable(dev);
mlx5_dev_interrupt_handler_uninstall(dev);
mlx5_txq_stop(dev);
mlx5_rxq_stop(dev);
for (mr = LIST_FIRST(&priv->mr); mr; mr = LIST_FIRST(&priv->mr))
priv_mr_release(priv, mr);
priv_flow_delete_drop_queue(priv);
mlx5_mr_release(mr);
mlx5_flow_delete_drop_queue(dev);
}
/**
* Enable traffic flows configured by control plane
*
* @param priv
* @param dev
* Pointer to Ethernet device private data.
* @param dev
* Pointer to Ethernet device structure.
@ -237,8 +241,9 @@ mlx5_dev_stop(struct rte_eth_dev *dev)
* 0 on success.
*/
int
priv_dev_traffic_enable(struct priv *priv, struct rte_eth_dev *dev)
mlx5_traffic_enable(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct rte_flow_item_eth bcast = {
.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
};
@ -356,40 +361,18 @@ error:
/**
* Disable traffic flows configured by control plane
*
* @param priv
* Pointer to Ethernet device private data.
* @param dev
* Pointer to Ethernet device structure.
* Pointer to Ethernet device private data.
*
* @return
* 0 on success.
*/
int
priv_dev_traffic_disable(struct priv *priv,
struct rte_eth_dev *dev __rte_unused)
mlx5_traffic_disable(struct rte_eth_dev *dev)
{
priv_flow_flush(priv, &priv->ctrl_flows);
return 0;
}
struct priv *priv = dev->data->dev_private;
/**
* Restart traffic flows configured by control plane
*
* @param priv
* Pointer to Ethernet device private data.
* @param dev
* Pointer to Ethernet device structure.
*
* @return
* 0 on success.
*/
int
priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
{
if (dev->data->dev_started) {
priv_dev_traffic_disable(priv, dev);
priv_dev_traffic_enable(priv, dev);
}
mlx5_flow_list_flush(dev, &priv->ctrl_flows);
return 0;
}
@ -397,7 +380,7 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
* Restart traffic flows configured by control plane
*
* @param dev
* Pointer to Ethernet device structure.
* Pointer to Ethernet device private data.
*
* @return
* 0 on success.
@ -405,8 +388,9 @@ priv_dev_traffic_restart(struct priv *priv, struct rte_eth_dev *dev)
int
mlx5_traffic_restart(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
priv_dev_traffic_restart(priv, dev);
if (dev->data->dev_started) {
mlx5_traffic_disable(dev);
mlx5_traffic_enable(dev);
}
return 0;
}

134
drivers/net/mlx5/mlx5_txq.c
View File

@ -91,15 +91,16 @@ txq_free_elts(struct mlx5_txq_ctrl *txq_ctrl)
/**
* Returns the per-port supported offloads.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* Supported Tx offloads.
*/
uint64_t
mlx5_priv_get_tx_port_offloads(struct priv *priv)
mlx5_get_tx_port_offloads(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
uint64_t offloads = (DEV_TX_OFFLOAD_MULTI_SEGS |
DEV_TX_OFFLOAD_VLAN_INSERT);
struct mlx5_dev_config *config = &priv->config;
@ -123,8 +124,8 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv)
/**
* Checks if the per-queue offload configuration is valid.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param offloads
* Per-queue offloads configuration.
*
@ -132,10 +133,10 @@ mlx5_priv_get_tx_port_offloads(struct priv *priv)
* 1 if the configuration is valid, 0 otherwise.
*/
static int
priv_is_tx_queue_offloads_allowed(struct priv *priv, uint64_t offloads)
mlx5_is_tx_queue_offloads_allowed(struct rte_eth_dev *dev, uint64_t offloads)
{
uint64_t port_offloads = priv->dev->data->dev_conf.txmode.offloads;
uint64_t port_supp_offloads = mlx5_priv_get_tx_port_offloads(priv);
uint64_t port_offloads = dev->data->dev_conf.txmode.offloads;
uint64_t port_supp_offloads = mlx5_get_tx_port_offloads(dev);
/* There are no Tx offloads which are per queue. */
if ((offloads & port_supp_offloads) != offloads)
@ -177,13 +178,13 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
* use the old API.
*/
if (!!(conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) &&
!priv_is_tx_queue_offloads_allowed(priv, conf->offloads)) {
!mlx5_is_tx_queue_offloads_allowed(dev, conf->offloads)) {
ret = ENOTSUP;
ERROR("%p: Tx queue offloads 0x%" PRIx64 " don't match port "
"offloads 0x%" PRIx64 " or supported offloads 0x%" PRIx64,
(void *)dev, conf->offloads,
dev->data->dev_conf.txmode.offloads,
mlx5_priv_get_tx_port_offloads(priv));
mlx5_get_tx_port_offloads(dev));
goto out;
}
if (desc <= MLX5_TX_COMP_THRESH) {
@ -206,14 +207,14 @@ mlx5_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
(void *)dev, idx, priv->txqs_n);
return -EOVERFLOW;
}
if (!mlx5_priv_txq_releasable(priv, idx)) {
if (!mlx5_txq_releasable(dev, idx)) {
ret = EBUSY;
ERROR("%p: unable to release queue index %u",
(void *)dev, idx);
goto out;
}
mlx5_priv_txq_release(priv, idx);
txq_ctrl = mlx5_priv_txq_new(priv, idx, desc, socket, conf);
mlx5_txq_release(dev, idx);
txq_ctrl = mlx5_txq_new(dev, idx, desc, socket, conf);
if (!txq_ctrl) {
ERROR("%p: unable to allocate queue index %u",
(void *)dev, idx);
@ -249,7 +250,7 @@ mlx5_tx_queue_release(void *dpdk_txq)
if ((*priv->txqs)[i] == txq) {
DEBUG("%p: removing TX queue %p from list",
(void *)priv->dev, (void *)txq_ctrl);
mlx5_priv_txq_release(priv, i);
mlx5_txq_release(priv->dev, i);
break;
}
}
@ -260,8 +261,8 @@ mlx5_tx_queue_release(void *dpdk_txq)
* Both primary and secondary process do mmap to make UAR address
* aligned.
*
* @param[in] priv
* Pointer to private structure.
* @param[in] dev
* Pointer to Ethernet device.
* @param fd
* Verbs file descriptor to map UAR pages.
*
@ -269,8 +270,9 @@ mlx5_tx_queue_release(void *dpdk_txq)
* 0 on success, errno value on failure.
*/
int
priv_tx_uar_remap(struct priv *priv, int fd)
mlx5_tx_uar_remap(struct rte_eth_dev *dev, int fd)
{
struct priv *priv = dev->data->dev_private;
unsigned int i, j;
uintptr_t pages[priv->txqs_n];
unsigned int pages_n = 0;
@ -356,8 +358,8 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst)
/**
* Create the Tx queue Verbs object.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* Queue index in DPDK Rx queue array
*
@ -365,8 +367,9 @@ is_empw_burst_func(eth_tx_burst_t tx_pkt_burst)
* The Verbs object initialised if it can be created.
*/
struct mlx5_txq_ibv *
mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
mlx5_txq_ibv_new(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_data *txq_data = (*priv->txqs)[idx];
struct mlx5_txq_ctrl *txq_ctrl =
container_of(txq_data, struct mlx5_txq_ctrl, txq);
@ -383,7 +386,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
struct mlx5dv_cq cq_info;
struct mlx5dv_obj obj;
const int desc = 1 << txq_data->elts_n;
eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev);
eth_tx_burst_t tx_pkt_burst = mlx5_select_tx_function(dev);
int ret = 0;
assert(txq_data);
@ -517,7 +520,7 @@ mlx5_priv_txq_ibv_new(struct priv *priv, uint16_t idx)
ERROR("Failed to retrieve UAR info, invalid libmlx5.so version");
goto error;
}
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev,
(void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt));
LIST_INSERT_HEAD(&priv->txqsibv, txq_ibv, next);
priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
@ -534,8 +537,8 @@ error:
/**
* Get an Tx queue Verbs object.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* Queue index in DPDK Rx queue array
*
@ -543,8 +546,9 @@ error:
* The Verbs object if it exists.
*/
struct mlx5_txq_ibv *
mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
mlx5_txq_ibv_get(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *txq_ctrl;
if (idx >= priv->txqs_n)
@ -554,7 +558,7 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
if (txq_ctrl->ibv) {
rte_atomic32_inc(&txq_ctrl->ibv->refcnt);
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)dev,
(void *)txq_ctrl->ibv,
rte_atomic32_read(&txq_ctrl->ibv->refcnt));
}
@ -564,8 +568,6 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
/**
* Release an Tx verbs queue object.
*
* @param priv
* Pointer to private structure.
* @param txq_ibv
* Verbs Tx queue object.
*
@ -573,11 +575,10 @@ mlx5_priv_txq_ibv_get(struct priv *priv, uint16_t idx)
* 0 on success, errno on failure.
*/
int
mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused,
struct mlx5_txq_ibv *txq_ibv)
mlx5_txq_ibv_release(struct mlx5_txq_ibv *txq_ibv)
{
assert(txq_ibv);
DEBUG("%p: Verbs Tx queue %p: refcnt %d", (void *)priv,
DEBUG("Verbs Tx queue %p: refcnt %d",
(void *)txq_ibv, rte_atomic32_read(&txq_ibv->refcnt));
if (rte_atomic32_dec_and_test(&txq_ibv->refcnt)) {
claim_zero(mlx5_glue->destroy_qp(txq_ibv->qp));
@ -592,14 +593,11 @@ mlx5_priv_txq_ibv_release(struct priv *priv __rte_unused,
/**
* Return true if a single reference exists on the object.
*
* @param priv
* Pointer to private structure.
* @param txq_ibv
* Verbs Tx queue object.
*/
int
mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused,
struct mlx5_txq_ibv *txq_ibv)
mlx5_txq_ibv_releasable(struct mlx5_txq_ibv *txq_ibv)
{
assert(txq_ibv);
return (rte_atomic32_read(&txq_ibv->refcnt) == 1);
@ -608,20 +606,21 @@ mlx5_priv_txq_ibv_releasable(struct priv *priv __rte_unused,
/**
* Verify the Verbs Tx queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_txq_ibv_verify(struct priv *priv)
mlx5_txq_ibv_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
int ret = 0;
struct mlx5_txq_ibv *txq_ibv;
LIST_FOREACH(txq_ibv, &priv->txqsibv, next) {
DEBUG("%p: Verbs Tx queue %p still referenced", (void *)priv,
DEBUG("%p: Verbs Tx queue %p still referenced", (void *)dev,
(void *)txq_ibv);
++ret;
}
@ -645,7 +644,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
unsigned int txq_inline;
unsigned int txqs_inline;
unsigned int inline_max_packet_sz;
eth_tx_burst_t tx_pkt_burst = priv_select_tx_function(priv, priv->dev);
eth_tx_burst_t tx_pkt_burst =
mlx5_select_tx_function(txq_ctrl->priv->dev);
int is_empw_func = is_empw_burst_func(tx_pkt_burst);
int tso = !!(txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_TCP_TSO);
@ -731,8 +731,8 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
/**
* Create a DPDK Tx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
* @param desc
@ -746,10 +746,10 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
* A DPDK queue object on success.
*/
struct mlx5_txq_ctrl *
mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
unsigned int socket,
const struct rte_eth_txconf *conf)
mlx5_txq_new(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
unsigned int socket, const struct rte_eth_txconf *conf)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *tmpl;
tmpl = rte_calloc_socket("TXQ", 1,
@ -773,7 +773,7 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
(struct rte_mbuf *(*)[1 << tmpl->txq.elts_n])(tmpl + 1);
tmpl->txq.stats.idx = idx;
rte_atomic32_inc(&tmpl->refcnt);
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
(void *)tmpl, rte_atomic32_read(&tmpl->refcnt));
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
return tmpl;
@ -782,8 +782,8 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
/**
* Get a Tx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -791,8 +791,9 @@ mlx5_priv_txq_new(struct priv *priv, uint16_t idx, uint16_t desc,
* A pointer to the queue if it exists.
*/
struct mlx5_txq_ctrl *
mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
mlx5_txq_get(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *ctrl = NULL;
if ((*priv->txqs)[idx]) {
@ -800,15 +801,15 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
txq);
unsigned int i;
mlx5_priv_txq_ibv_get(priv, idx);
mlx5_txq_ibv_get(dev, idx);
for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
if (ctrl->txq.mp2mr[i])
claim_nonzero
(priv_mr_get(priv,
(mlx5_mr_get(dev,
ctrl->txq.mp2mr[i]->mp));
}
rte_atomic32_inc(&ctrl->refcnt);
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
(void *)ctrl, rte_atomic32_read(&ctrl->refcnt));
}
return ctrl;
@ -817,8 +818,8 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
/**
* Release a Tx queue.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -826,8 +827,9 @@ mlx5_priv_txq_get(struct priv *priv, uint16_t idx)
* 0 on success, errno on failure.
*/
int
mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
mlx5_txq_release(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
unsigned int i;
struct mlx5_txq_ctrl *txq;
size_t page_size = sysconf(_SC_PAGESIZE);
@ -835,18 +837,18 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
if (!(*priv->txqs)[idx])
return 0;
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
DEBUG("%p: Tx queue %p: refcnt %d", (void *)priv,
DEBUG("%p: Tx queue %p: refcnt %d", (void *)dev,
(void *)txq, rte_atomic32_read(&txq->refcnt));
if (txq->ibv) {
int ret;
ret = mlx5_priv_txq_ibv_release(priv, txq->ibv);
ret = mlx5_txq_ibv_release(txq->ibv);
if (!ret)
txq->ibv = NULL;
}
for (i = 0; i != MLX5_PMD_TX_MP_CACHE; ++i) {
if (txq->txq.mp2mr[i]) {
priv_mr_release(priv, txq->txq.mp2mr[i]);
mlx5_mr_release(txq->txq.mp2mr[i]);
txq->txq.mp2mr[i] = NULL;
}
}
@ -866,8 +868,8 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
/**
* Verify if the queue can be released.
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
* @param idx
* TX queue index.
*
@ -875,8 +877,9 @@ mlx5_priv_txq_release(struct priv *priv, uint16_t idx)
* 1 if the queue can be released.
*/
int
mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx)
mlx5_txq_releasable(struct rte_eth_dev *dev, uint16_t idx)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *txq;
if (!(*priv->txqs)[idx])
@ -888,20 +891,21 @@ mlx5_priv_txq_releasable(struct priv *priv, uint16_t idx)
/**
* Verify the Tx Queue list is empty
*
* @param priv
* Pointer to private structure.
* @param dev
* Pointer to Ethernet device.
*
* @return
* The number of object not released.
*/
int
mlx5_priv_txq_verify(struct priv *priv)
mlx5_txq_verify(struct rte_eth_dev *dev)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *txq;
int ret = 0;
LIST_FOREACH(txq, &priv->txqsctrl, next) {
DEBUG("%p: Tx Queue %p still referenced", (void *)priv,
DEBUG("%p: Tx Queue %p still referenced", (void *)dev,
(void *)txq);
++ret;
}

77
drivers/net/mlx5/mlx5_vlan.c
View File

@ -79,55 +79,11 @@ mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
++priv->vlan_filter_n;
}
if (dev->data->dev_started)
priv_dev_traffic_restart(priv, dev);
mlx5_traffic_restart(dev);
out:
return ret;
}
/**
* Set/reset VLAN stripping for a specific queue.
*
* @param priv
* Pointer to private structure.
* @param idx
* RX queue index.
* @param on
* Enable/disable VLAN stripping.
*/
static void
priv_vlan_strip_queue_set(struct priv *priv, uint16_t idx, int on)
{
struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
struct mlx5_rxq_ctrl *rxq_ctrl =
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
struct ibv_wq_attr mod;
uint16_t vlan_offloads =
(on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
0;
int err;
DEBUG("set VLAN offloads 0x%x for port %d queue %d",
vlan_offloads, rxq->port_id, idx);
if (!rxq_ctrl->ibv) {
/* Update related bits in RX queue. */
rxq->vlan_strip = !!on;
return;
}
mod = (struct ibv_wq_attr){
.attr_mask = IBV_WQ_ATTR_FLAGS,
.flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
.flags = vlan_offloads,
};
err = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
if (err) {
ERROR("%p: failed to modified stripping mode: %s",
(void *)priv, strerror(err));
return;
}
/* Update related bits in RX queue. */
rxq->vlan_strip = !!on;
}
/**
* Callback to set/reset VLAN stripping for a specific queue.
*
@ -142,6 +98,14 @@ void
mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
{
struct priv *priv = dev->data->dev_private;
struct mlx5_rxq_data *rxq = (*priv->rxqs)[queue];
struct mlx5_rxq_ctrl *rxq_ctrl =
container_of(rxq, struct mlx5_rxq_ctrl, rxq);
struct ibv_wq_attr mod;
uint16_t vlan_offloads =
(on ? IBV_WQ_FLAGS_CVLAN_STRIPPING : 0) |
0;
int err;
/* Validate hw support */
if (!priv->config.hw_vlan_strip) {
@ -153,7 +117,26 @@ mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on)
ERROR("VLAN stripping, invalid queue number %d", queue);
return;
}
priv_vlan_strip_queue_set(priv, queue, on);
DEBUG("set VLAN offloads 0x%x for port %d queue %d",
vlan_offloads, rxq->port_id, queue);
if (!rxq_ctrl->ibv) {
/* Update related bits in RX queue. */
rxq->vlan_strip = !!on;
return;
}
mod = (struct ibv_wq_attr){
.attr_mask = IBV_WQ_ATTR_FLAGS,
.flags_mask = IBV_WQ_FLAGS_CVLAN_STRIPPING,
.flags = vlan_offloads,
};
err = mlx5_glue->modify_wq(rxq_ctrl->ibv->wq, &mod);
if (err) {
ERROR("%p: failed to modified stripping mode: %s",
(void *)dev, strerror(err));
return;
}
/* Update related bits in RX queue. */
rxq->vlan_strip = !!on;
}
/**
@ -180,7 +163,7 @@ mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask)
}
/* Run on every RX queue and set/reset VLAN stripping. */
for (i = 0; (i != priv->rxqs_n); i++)
priv_vlan_strip_queue_set(priv, i, hw_vlan_strip);
mlx5_vlan_strip_queue_set(dev, i, hw_vlan_strip);
}
return 0;
}