d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
0f3ba0d4a8
numam-dpdk/drivers/net/mlx5/mlx5_flow_hw.c
Michael Baum d37435dc3f net/mlx5: assert for enough space in counter rings 
There is a by-design assumption in the code that the global counter
rings can contain all the port counters.
So, enqueuing to these global rings should always succeed.

Add assertions to help for debugging this assumption.

In addition, change mlx5_hws_cnt_pool_put() function to return void due
to those assumptions.

Signed-off-by: Michael Baum <michaelba@nvidia.com>
Acked-by: Matan Azrad <matan@nvidia.com>
Acked-by: Xiaoyu Min <jackmin@nvidia.com>
2022-11-10 18:15:45 +01:00

9463 lines
277 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright (c) 2022 NVIDIA Corporation & Affiliates
*/
#include <rte_flow.h>
#include <mlx5_malloc.h>
#include "mlx5_defs.h"
#include "mlx5_flow.h"
#include "mlx5_rx.h"
#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H)
#include "mlx5_hws_cnt.h"
/* The maximum actions support in the flow. */
#define MLX5_HW_MAX_ACTS 16
/*
* The default ipool threshold value indicates which per_core_cache
* value to set.
*/
#define MLX5_HW_IPOOL_SIZE_THRESHOLD (1 << 19)
/* The default min local cache size. */
#define MLX5_HW_IPOOL_CACHE_MIN (1 << 9)
/* Default push burst threshold. */
#define BURST_THR 32u
/* Default queue to flush the flows. */
#define MLX5_DEFAULT_FLUSH_QUEUE 0
/* Maximum number of rules in control flow tables. */
#define MLX5_HW_CTRL_FLOW_NB_RULES (4096)
/* Lowest flow group usable by an application if group translation is done. */
#define MLX5_HW_LOWEST_USABLE_GROUP (1)
/* Maximum group index usable by user applications for transfer flows. */
#define MLX5_HW_MAX_TRANSFER_GROUP (UINT32_MAX - 1)
/* Maximum group index usable by user applications for egress flows. */
#define MLX5_HW_MAX_EGRESS_GROUP (UINT32_MAX - 1)
/* Lowest priority for HW root table. */
#define MLX5_HW_LOWEST_PRIO_ROOT 15
/* Lowest priority for HW non-root table. */
#define MLX5_HW_LOWEST_PRIO_NON_ROOT (UINT32_MAX)
/* Priorities for Rx control flow rules. */
#define MLX5_HW_CTRL_RX_PRIO_L2 (MLX5_HW_LOWEST_PRIO_ROOT)
#define MLX5_HW_CTRL_RX_PRIO_L3 (MLX5_HW_LOWEST_PRIO_ROOT - 1)
#define MLX5_HW_CTRL_RX_PRIO_L4 (MLX5_HW_LOWEST_PRIO_ROOT - 2)
#define MLX5_HW_VLAN_PUSH_TYPE_IDX 0
#define MLX5_HW_VLAN_PUSH_VID_IDX 1
#define MLX5_HW_VLAN_PUSH_PCP_IDX 2
static int flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev);
static int flow_hw_translate_group(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
uint32_t group,
uint32_t *table_group,
struct rte_flow_error *error);
static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
struct mlx5_hw_q_job *job,
struct mlx5_action_construct_data *act_data,
const struct mlx5_hw_actions *hw_acts,
const struct rte_flow_action *action);
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev);
static __rte_always_inline uint32_t flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev);
const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops;
/* DR action flags with different table. */
static uint32_t mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_MAX]
[MLX5DR_TABLE_TYPE_MAX] = {
{
MLX5DR_ACTION_FLAG_ROOT_RX,
MLX5DR_ACTION_FLAG_ROOT_TX,
MLX5DR_ACTION_FLAG_ROOT_FDB,
},
{
MLX5DR_ACTION_FLAG_HWS_RX,
MLX5DR_ACTION_FLAG_HWS_TX,
MLX5DR_ACTION_FLAG_HWS_FDB,
},
};
/* Ethernet item spec for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_spec = {
.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item mask for promiscuous mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_promisc_mask = {
.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item spec for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_spec = {
.dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item mask for all multicast mode. */
static const struct rte_flow_item_eth ctrl_rx_eth_mcast_mask = {
.dst.addr_bytes = "\x01\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item spec for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_spec = {
.dst.addr_bytes = "\x01\x00\x5e\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item mask for IPv4 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv4_mcast_mask = {
.dst.addr_bytes = "\xff\xff\xff\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item spec for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_spec = {
.dst.addr_bytes = "\x33\x33\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item mask for IPv6 multicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_ipv6_mcast_mask = {
.dst.addr_bytes = "\xff\xff\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item mask for unicast traffic. */
static const struct rte_flow_item_eth ctrl_rx_eth_dmac_mask = {
.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/* Ethernet item spec for broadcast. */
static const struct rte_flow_item_eth ctrl_rx_eth_bcast_spec = {
.dst.addr_bytes = "\xff\xff\xff\xff\xff\xff",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
/**
* Set rxq flag.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] enable
* Flag to enable or not.
*/
static void
flow_hw_rxq_flag_set(struct rte_eth_dev *dev, bool enable)
{
struct mlx5_priv *priv = dev->data->dev_private;
unsigned int i;
if ((!priv->mark_enabled && !enable) ||
(priv->mark_enabled && enable))
return;
for (i = 0; i < priv->rxqs_n; ++i) {
struct mlx5_rxq_ctrl *rxq_ctrl = mlx5_rxq_ctrl_get(dev, i);
/* With RXQ start/stop feature, RXQ might be stopped. */
if (!rxq_ctrl)
continue;
rxq_ctrl->rxq.mark = enable;
}
priv->mark_enabled = enable;
}
/**
* Set the hash fields according to the @p rss_desc information.
*
* @param[in] rss_desc
* Pointer to the mlx5_flow_rss_desc.
* @param[out] hash_fields
* Pointer to the RSS hash fields.
*/
static void
flow_hw_hashfields_set(struct mlx5_flow_rss_desc *rss_desc,
uint64_t *hash_fields)
{
uint64_t fields = 0;
int rss_inner = 0;
uint64_t rss_types = rte_eth_rss_hf_refine(rss_desc->types);
#ifdef HAVE_IBV_DEVICE_TUNNEL_SUPPORT
if (rss_desc->level >= 2)
rss_inner = 1;
#endif
if (rss_types & MLX5_IPV4_LAYER_TYPES) {
if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
fields |= IBV_RX_HASH_SRC_IPV4;
else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
fields |= IBV_RX_HASH_DST_IPV4;
else
fields |= MLX5_IPV4_IBV_RX_HASH;
} else if (rss_types & MLX5_IPV6_LAYER_TYPES) {
if (rss_types & RTE_ETH_RSS_L3_SRC_ONLY)
fields |= IBV_RX_HASH_SRC_IPV6;
else if (rss_types & RTE_ETH_RSS_L3_DST_ONLY)
fields |= IBV_RX_HASH_DST_IPV6;
else
fields |= MLX5_IPV6_IBV_RX_HASH;
}
if (rss_types & RTE_ETH_RSS_UDP) {
if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
fields |= IBV_RX_HASH_SRC_PORT_UDP;
else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
fields |= IBV_RX_HASH_DST_PORT_UDP;
else
fields |= MLX5_UDP_IBV_RX_HASH;
} else if (rss_types & RTE_ETH_RSS_TCP) {
if (rss_types & RTE_ETH_RSS_L4_SRC_ONLY)
fields |= IBV_RX_HASH_SRC_PORT_TCP;
else if (rss_types & RTE_ETH_RSS_L4_DST_ONLY)
fields |= IBV_RX_HASH_DST_PORT_TCP;
else
fields |= MLX5_TCP_IBV_RX_HASH;
}
if (rss_types & RTE_ETH_RSS_ESP)
fields |= IBV_RX_HASH_IPSEC_SPI;
if (rss_inner)
fields |= IBV_RX_HASH_INNER;
*hash_fields = fields;
}
/**
* Generate the pattern item flags.
* Will be used for shared RSS action.
*
* @param[in] items
* Pointer to the list of items.
*
* @return
* Item flags.
*/
static uint64_t
flow_hw_rss_item_flags_get(const struct rte_flow_item items[])
{
uint64_t item_flags = 0;
uint64_t last_item = 0;
for (; items->type != RTE_FLOW_ITEM_TYPE_END; items++) {
int tunnel = !!(item_flags & MLX5_FLOW_LAYER_TUNNEL);
int item_type = items->type;
switch (item_type) {
case RTE_FLOW_ITEM_TYPE_IPV4:
last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV4 :
MLX5_FLOW_LAYER_OUTER_L3_IPV4;
break;
case RTE_FLOW_ITEM_TYPE_IPV6:
last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L3_IPV6 :
MLX5_FLOW_LAYER_OUTER_L3_IPV6;
break;
case RTE_FLOW_ITEM_TYPE_TCP:
last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_TCP :
MLX5_FLOW_LAYER_OUTER_L4_TCP;
break;
case RTE_FLOW_ITEM_TYPE_UDP:
last_item = tunnel ? MLX5_FLOW_LAYER_INNER_L4_UDP :
MLX5_FLOW_LAYER_OUTER_L4_UDP;
break;
case RTE_FLOW_ITEM_TYPE_GRE:
last_item = MLX5_FLOW_LAYER_GRE;
break;
case RTE_FLOW_ITEM_TYPE_NVGRE:
last_item = MLX5_FLOW_LAYER_GRE;
break;
case RTE_FLOW_ITEM_TYPE_VXLAN:
last_item = MLX5_FLOW_LAYER_VXLAN;
break;
case RTE_FLOW_ITEM_TYPE_VXLAN_GPE:
last_item = MLX5_FLOW_LAYER_VXLAN_GPE;
break;
case RTE_FLOW_ITEM_TYPE_GENEVE:
last_item = MLX5_FLOW_LAYER_GENEVE;
break;
case RTE_FLOW_ITEM_TYPE_MPLS:
last_item = MLX5_FLOW_LAYER_MPLS;
break;
case RTE_FLOW_ITEM_TYPE_GTP:
last_item = MLX5_FLOW_LAYER_GTP;
break;
default:
break;
}
item_flags |= last_item;
}
return item_flags;
}
/**
* Register destination table DR jump action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] table_attr
* Pointer to the flow attributes.
* @param[in] dest_group
* The destination group ID.
* @param[out] error
* Pointer to error structure.
*
* @return
* Table on success, NULL otherwise and rte_errno is set.
*/
static struct mlx5_hw_jump_action *
flow_hw_jump_action_register(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
uint32_t dest_group,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_attr jattr = cfg->attr.flow_attr;
struct mlx5_flow_group *grp;
struct mlx5_flow_cb_ctx ctx = {
.dev = dev,
.error = error,
.data = &jattr,
};
struct mlx5_list_entry *ge;
uint32_t target_group;
target_group = dest_group;
if (flow_hw_translate_group(dev, cfg, dest_group, &target_group, error))
return NULL;
jattr.group = target_group;
ge = mlx5_hlist_register(priv->sh->flow_tbls, target_group, &ctx);
if (!ge)
return NULL;
grp = container_of(ge, struct mlx5_flow_group, entry);
return &grp->jump;
}
/**
* Release jump action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] jump
* Pointer to the jump action.
*/
static void
flow_hw_jump_release(struct rte_eth_dev *dev, struct mlx5_hw_jump_action *jump)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_group *grp;
grp = container_of
(jump, struct mlx5_flow_group, jump);
mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
}
/**
* Register queue/RSS action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] hws_flags
* DR action flags.
* @param[in] action
* rte flow action.
*
* @return
* Table on success, NULL otherwise and rte_errno is set.
*/
static inline struct mlx5_hrxq*
flow_hw_tir_action_register(struct rte_eth_dev *dev,
uint32_t hws_flags,
const struct rte_flow_action *action)
{
struct mlx5_flow_rss_desc rss_desc = {
.hws_flags = hws_flags,
};
struct mlx5_hrxq *hrxq;
if (action->type == RTE_FLOW_ACTION_TYPE_QUEUE) {
const struct rte_flow_action_queue *queue = action->conf;
rss_desc.const_q = &queue->index;
rss_desc.queue_num = 1;
} else {
const struct rte_flow_action_rss *rss = action->conf;
rss_desc.queue_num = rss->queue_num;
rss_desc.const_q = rss->queue;
memcpy(rss_desc.key,
!rss->key ? rss_hash_default_key : rss->key,
MLX5_RSS_HASH_KEY_LEN);
rss_desc.key_len = MLX5_RSS_HASH_KEY_LEN;
rss_desc.types = !rss->types ? RTE_ETH_RSS_IP : rss->types;
flow_hw_hashfields_set(&rss_desc, &rss_desc.hash_fields);
flow_dv_action_rss_l34_hash_adjust(rss->types,
&rss_desc.hash_fields);
if (rss->level > 1) {
rss_desc.hash_fields |= IBV_RX_HASH_INNER;
rss_desc.tunnel = 1;
}
}
hrxq = mlx5_hrxq_get(dev, &rss_desc);
return hrxq;
}
static __rte_always_inline int
flow_hw_ct_compile(struct rte_eth_dev *dev,
uint32_t queue, uint32_t idx,
struct mlx5dr_rule_action *rule_act)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_action *ct;
ct = mlx5_ipool_get(priv->hws_ctpool->cts, MLX5_ACTION_CTX_CT_GET_IDX(idx));
if (!ct || mlx5_aso_ct_available(priv->sh, queue, ct))
return -1;
rule_act->action = priv->hws_ctpool->dr_action;
rule_act->aso_ct.offset = ct->offset;
rule_act->aso_ct.direction = ct->is_original ?
MLX5DR_ACTION_ASO_CT_DIRECTION_INITIATOR :
MLX5DR_ACTION_ASO_CT_DIRECTION_RESPONDER;
return 0;
}
/**
* Destroy DR actions created by action template.
*
* For DR actions created during table creation's action translate.
* Need to destroy the DR action when destroying the table.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
*/
static void
__flow_hw_action_template_destroy(struct rte_eth_dev *dev,
struct mlx5_hw_actions *acts)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_action_construct_data *data;
while (!LIST_EMPTY(&acts->act_list)) {
data = LIST_FIRST(&acts->act_list);
LIST_REMOVE(data, next);
mlx5_ipool_free(priv->acts_ipool, data->idx);
}
if (acts->jump) {
struct mlx5_flow_group *grp;
grp = container_of
(acts->jump, struct mlx5_flow_group, jump);
mlx5_hlist_unregister(priv->sh->flow_tbls, &grp->entry);
acts->jump = NULL;
}
if (acts->tir) {
mlx5_hrxq_release(dev, acts->tir->idx);
acts->tir = NULL;
}
if (acts->encap_decap) {
if (acts->encap_decap->action)
mlx5dr_action_destroy(acts->encap_decap->action);
mlx5_free(acts->encap_decap);
acts->encap_decap = NULL;
}
if (acts->mhdr) {
if (acts->mhdr->action)
mlx5dr_action_destroy(acts->mhdr->action);
mlx5_free(acts->mhdr);
}
if (mlx5_hws_cnt_id_valid(acts->cnt_id)) {
mlx5_hws_cnt_shared_put(priv->hws_cpool, &acts->cnt_id);
acts->cnt_id = 0;
}
if (acts->mtr_id) {
mlx5_ipool_free(priv->hws_mpool->idx_pool, acts->mtr_id);
acts->mtr_id = 0;
}
}
/**
* Append dynamic action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline struct mlx5_action_construct_data *
__flow_hw_act_data_alloc(struct mlx5_priv *priv,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst)
{
struct mlx5_action_construct_data *act_data;
uint32_t idx = 0;
act_data = mlx5_ipool_zmalloc(priv->acts_ipool, &idx);
if (!act_data)
return NULL;
act_data->idx = idx;
act_data->type = type;
act_data->action_src = action_src;
act_data->action_dst = action_dst;
return act_data;
}
/**
* Append dynamic action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
__flow_hw_act_data_general_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst)
{
struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
/**
* Append dynamic encap action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
* @param[in] len
* Length of the data to be updated.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
__flow_hw_act_data_encap_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst,
uint16_t len)
{
struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
act_data->encap.len = len;
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
static __rte_always_inline int
__flow_hw_act_data_hdr_modify_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst,
uint16_t mhdr_cmds_off,
uint16_t mhdr_cmds_end,
bool shared,
struct field_modify_info *field,
struct field_modify_info *dcopy,
uint32_t *mask)
{
struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
act_data->modify_header.mhdr_cmds_off = mhdr_cmds_off;
act_data->modify_header.mhdr_cmds_end = mhdr_cmds_end;
act_data->modify_header.shared = shared;
rte_memcpy(act_data->modify_header.field, field,
sizeof(*field) * MLX5_ACT_MAX_MOD_FIELDS);
rte_memcpy(act_data->modify_header.dcopy, dcopy,
sizeof(*dcopy) * MLX5_ACT_MAX_MOD_FIELDS);
rte_memcpy(act_data->modify_header.mask, mask,
sizeof(*mask) * MLX5_ACT_MAX_MOD_FIELDS);
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
/**
* Append shared RSS action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
* @param[in] idx
* Shared RSS index.
* @param[in] rss
* Pointer to the shared RSS info.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
__flow_hw_act_data_shared_rss_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst,
uint32_t idx,
struct mlx5_shared_action_rss *rss)
{
struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
act_data->shared_rss.level = rss->origin.level;
act_data->shared_rss.types = !rss->origin.types ? RTE_ETH_RSS_IP :
rss->origin.types;
act_data->shared_rss.idx = idx;
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
/**
* Append shared counter action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
* @param[in] cnt_id
* Shared counter id.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
__flow_hw_act_data_shared_cnt_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst,
cnt_id_t cnt_id)
{
struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
act_data->type = type;
act_data->shared_counter.id = cnt_id;
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
/**
* Append shared meter_mark action to the dynamic action list.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] type
* Action type.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
* @param[in] mtr_id
* Shared meter id.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
__flow_hw_act_data_shared_mtr_append(struct mlx5_priv *priv,
struct mlx5_hw_actions *acts,
enum rte_flow_action_type type,
uint16_t action_src,
uint16_t action_dst,
cnt_id_t mtr_id)
{ struct mlx5_action_construct_data *act_data;
act_data = __flow_hw_act_data_alloc(priv, type, action_src, action_dst);
if (!act_data)
return -1;
act_data->type = type;
act_data->shared_meter.id = mtr_id;
LIST_INSERT_HEAD(&acts->act_list, act_data, next);
return 0;
}
/**
* Translate shared indirect action.
*
* @param[in] dev
* Pointer to the rte_eth_dev data structure.
* @param[in] action
* Pointer to the shared indirect rte_flow action.
* @param[in] acts
* Pointer to the template HW steering DR actions.
* @param[in] action_src
* Offset of source rte flow action.
* @param[in] action_dst
* Offset of destination DR action.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
flow_hw_shared_action_translate(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
struct mlx5_hw_actions *acts,
uint16_t action_src,
uint16_t action_dst)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_shared_action_rss *shared_rss;
uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t idx = act_idx &
((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
switch (type) {
case MLX5_INDIRECT_ACTION_TYPE_RSS:
shared_rss = mlx5_ipool_get
(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
if (!shared_rss || __flow_hw_act_data_shared_rss_append
(priv, acts,
(enum rte_flow_action_type)MLX5_RTE_FLOW_ACTION_TYPE_RSS,
action_src, action_dst, idx, shared_rss))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_COUNT:
if (__flow_hw_act_data_shared_cnt_append(priv, acts,
(enum rte_flow_action_type)
MLX5_RTE_FLOW_ACTION_TYPE_COUNT,
action_src, action_dst, act_idx))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_AGE:
/* Not supported, prevent by validate function. */
MLX5_ASSERT(0);
break;
case MLX5_INDIRECT_ACTION_TYPE_CT:
if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE,
idx, &acts->rule_acts[action_dst]))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
if (__flow_hw_act_data_shared_mtr_append(priv, acts,
(enum rte_flow_action_type)
MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK,
action_src, action_dst, idx))
return -1;
break;
default:
DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
break;
}
return 0;
}
static __rte_always_inline bool
flow_hw_action_modify_field_is_shared(const struct rte_flow_action *action,
const struct rte_flow_action *mask)
{
const struct rte_flow_action_modify_field *v = action->conf;
const struct rte_flow_action_modify_field *m = mask->conf;
if (v->src.field == RTE_FLOW_FIELD_VALUE) {
uint32_t j;
if (m == NULL)
return false;
for (j = 0; j < RTE_DIM(m->src.value); ++j) {
/*
* Immediate value is considered to be masked
* (and thus shared by all flow rules), if mask
* is non-zero. Partial mask over immediate value
* is not allowed.
*/
if (m->src.value[j])
return true;
}
return false;
}
if (v->src.field == RTE_FLOW_FIELD_POINTER)
return m->src.pvalue != NULL;
/*
* Source field types other than VALUE and
* POINTER are always shared.
*/
return true;
}
static __rte_always_inline bool
flow_hw_should_insert_nop(const struct mlx5_hw_modify_header_action *mhdr,
const struct mlx5_modification_cmd *cmd)
{
struct mlx5_modification_cmd last_cmd = { { 0 } };
struct mlx5_modification_cmd new_cmd = { { 0 } };
const uint32_t cmds_num = mhdr->mhdr_cmds_num;
unsigned int last_type;
bool should_insert = false;
if (cmds_num == 0)
return false;
last_cmd = *(&mhdr->mhdr_cmds[cmds_num - 1]);
last_cmd.data0 = rte_be_to_cpu_32(last_cmd.data0);
last_cmd.data1 = rte_be_to_cpu_32(last_cmd.data1);
last_type = last_cmd.action_type;
new_cmd = *cmd;
new_cmd.data0 = rte_be_to_cpu_32(new_cmd.data0);
new_cmd.data1 = rte_be_to_cpu_32(new_cmd.data1);
switch (new_cmd.action_type) {
case MLX5_MODIFICATION_TYPE_SET:
case MLX5_MODIFICATION_TYPE_ADD:
if (last_type == MLX5_MODIFICATION_TYPE_SET ||
last_type == MLX5_MODIFICATION_TYPE_ADD)
should_insert = new_cmd.field == last_cmd.field;
else if (last_type == MLX5_MODIFICATION_TYPE_COPY)
should_insert = new_cmd.field == last_cmd.dst_field;
else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
should_insert = false;
else
MLX5_ASSERT(false); /* Other types are not supported. */
break;
case MLX5_MODIFICATION_TYPE_COPY:
if (last_type == MLX5_MODIFICATION_TYPE_SET ||
last_type == MLX5_MODIFICATION_TYPE_ADD)
should_insert = (new_cmd.field == last_cmd.field ||
new_cmd.dst_field == last_cmd.field);
else if (last_type == MLX5_MODIFICATION_TYPE_COPY)
should_insert = (new_cmd.field == last_cmd.dst_field ||
new_cmd.dst_field == last_cmd.dst_field);
else if (last_type == MLX5_MODIFICATION_TYPE_NOP)
should_insert = false;
else
MLX5_ASSERT(false); /* Other types are not supported. */
break;
default:
/* Other action types should be rejected on AT validation. */
MLX5_ASSERT(false);
break;
}
return should_insert;
}
static __rte_always_inline int
flow_hw_mhdr_cmd_nop_append(struct mlx5_hw_modify_header_action *mhdr)
{
struct mlx5_modification_cmd *nop;
uint32_t num = mhdr->mhdr_cmds_num;
if (num + 1 >= MLX5_MHDR_MAX_CMD)
return -ENOMEM;
nop = mhdr->mhdr_cmds + num;
nop->data0 = 0;
nop->action_type = MLX5_MODIFICATION_TYPE_NOP;
nop->data0 = rte_cpu_to_be_32(nop->data0);
nop->data1 = 0;
mhdr->mhdr_cmds_num = num + 1;
return 0;
}
static __rte_always_inline int
flow_hw_mhdr_cmd_append(struct mlx5_hw_modify_header_action *mhdr,
struct mlx5_modification_cmd *cmd)
{
uint32_t num = mhdr->mhdr_cmds_num;
if (num + 1 >= MLX5_MHDR_MAX_CMD)
return -ENOMEM;
mhdr->mhdr_cmds[num] = *cmd;
mhdr->mhdr_cmds_num = num + 1;
return 0;
}
static __rte_always_inline int
flow_hw_converted_mhdr_cmds_append(struct mlx5_hw_modify_header_action *mhdr,
struct mlx5_flow_dv_modify_hdr_resource *resource)
{
uint32_t idx;
int ret;
for (idx = 0; idx < resource->actions_num; ++idx) {
struct mlx5_modification_cmd *src = &resource->actions[idx];
if (flow_hw_should_insert_nop(mhdr, src)) {
ret = flow_hw_mhdr_cmd_nop_append(mhdr);
if (ret)
return ret;
}
ret = flow_hw_mhdr_cmd_append(mhdr, src);
if (ret)
return ret;
}
return 0;
}
static __rte_always_inline void
flow_hw_modify_field_init(struct mlx5_hw_modify_header_action *mhdr,
struct rte_flow_actions_template *at)
{
memset(mhdr, 0, sizeof(*mhdr));
/* Modify header action without any commands is shared by default. */
mhdr->shared = true;
mhdr->pos = at->mhdr_off;
}
static __rte_always_inline int
flow_hw_modify_field_compile(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_action *action_start, /* Start of AT actions. */
const struct rte_flow_action *action, /* Current action from AT. */
const struct rte_flow_action *action_mask, /* Current mask from AT. */
struct mlx5_hw_actions *acts,
struct mlx5_hw_modify_header_action *mhdr,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_action_modify_field *conf = action->conf;
union {
struct mlx5_flow_dv_modify_hdr_resource resource;
uint8_t data[sizeof(struct mlx5_flow_dv_modify_hdr_resource) +
sizeof(struct mlx5_modification_cmd) * MLX5_MHDR_MAX_CMD];
} dummy;
struct mlx5_flow_dv_modify_hdr_resource *resource;
struct rte_flow_item item = {
.spec = NULL,
.mask = NULL
};
struct field_modify_info field[MLX5_ACT_MAX_MOD_FIELDS] = {
{0, 0, MLX5_MODI_OUT_NONE} };
struct field_modify_info dcopy[MLX5_ACT_MAX_MOD_FIELDS] = {
{0, 0, MLX5_MODI_OUT_NONE} };
uint32_t mask[MLX5_ACT_MAX_MOD_FIELDS] = { 0 };
uint32_t type, value = 0;
uint16_t cmds_start, cmds_end;
bool shared;
int ret;
/*
* Modify header action is shared if previous modify_field actions
* are shared and currently compiled action is shared.
*/
shared = flow_hw_action_modify_field_is_shared(action, action_mask);
mhdr->shared &= shared;
if (conf->src.field == RTE_FLOW_FIELD_POINTER ||
conf->src.field == RTE_FLOW_FIELD_VALUE) {
type = conf->operation == RTE_FLOW_MODIFY_SET ? MLX5_MODIFICATION_TYPE_SET :
MLX5_MODIFICATION_TYPE_ADD;
/* For SET/ADD fill the destination field (field) first. */
mlx5_flow_field_id_to_modify_info(&conf->dst, field, mask,
conf->width, dev,
attr, error);
item.spec = conf->src.field == RTE_FLOW_FIELD_POINTER ?
(void *)(uintptr_t)conf->src.pvalue :
(void *)(uintptr_t)&conf->src.value;
if (conf->dst.field == RTE_FLOW_FIELD_META ||
conf->dst.field == RTE_FLOW_FIELD_TAG ||
conf->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
conf->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
value = *(const unaligned_uint32_t *)item.spec;
value = rte_cpu_to_be_32(value);
item.spec = &value;
} else if (conf->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
/*
* QFI is passed as an uint8_t integer, but it is accessed through
* a 2nd least significant byte of a 32-bit field in modify header command.
*/
value = *(const uint8_t *)item.spec;
value = rte_cpu_to_be_32(value << 8);
item.spec = &value;
}
} else {
type = MLX5_MODIFICATION_TYPE_COPY;
/* For COPY fill the destination field (dcopy) without mask. */
mlx5_flow_field_id_to_modify_info(&conf->dst, dcopy, NULL,
conf->width, dev,
attr, error);
/* Then construct the source field (field) with mask. */
mlx5_flow_field_id_to_modify_info(&conf->src, field, mask,
conf->width, dev,
attr, error);
}
item.mask = &mask;
memset(&dummy, 0, sizeof(dummy));
resource = &dummy.resource;
ret = flow_dv_convert_modify_action(&item, field, dcopy, resource, type, error);
if (ret)
return ret;
MLX5_ASSERT(resource->actions_num > 0);
/*
* If previous modify field action collide with this one, then insert NOP command.
* This NOP command will not be a part of action's command range used to update commands
* on rule creation.
*/
if (flow_hw_should_insert_nop(mhdr, &resource->actions[0])) {
ret = flow_hw_mhdr_cmd_nop_append(mhdr);
if (ret)
return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "too many modify field operations specified");
}
cmds_start = mhdr->mhdr_cmds_num;
ret = flow_hw_converted_mhdr_cmds_append(mhdr, resource);
if (ret)
return rte_flow_error_set(error, ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "too many modify field operations specified");
cmds_end = mhdr->mhdr_cmds_num;
if (shared)
return 0;
ret = __flow_hw_act_data_hdr_modify_append(priv, acts, RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
action - action_start, mhdr->pos,
cmds_start, cmds_end, shared,
field, dcopy, mask);
if (ret)
return rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "not enough memory to store modify field metadata");
return 0;
}
static int
flow_hw_represented_port_compile(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
const struct rte_flow_action *action_start,
const struct rte_flow_action *action,
const struct rte_flow_action *action_mask,
struct mlx5_hw_actions *acts,
uint16_t action_dst,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_action_ethdev *v = action->conf;
const struct rte_flow_action_ethdev *m = action_mask->conf;
int ret;
if (!attr->group)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR, NULL,
"represented_port action cannot"
" be used on group 0");
if (!attr->transfer)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER,
NULL,
"represented_port action requires"
" transfer attribute");
if (attr->ingress || attr->egress)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR, NULL,
"represented_port action cannot"
" be used with direction attributes");
if (!priv->master)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"represented_port acton must"
" be used on proxy port");
if (m && !!m->port_id) {
struct mlx5_priv *port_priv;
if (!v)
return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
action, "port index was not provided");
port_priv = mlx5_port_to_eswitch_info(v->port_id, false);
if (port_priv == NULL)
return rte_flow_error_set
(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"port does not exist or unable to"
" obtain E-Switch info for port");
MLX5_ASSERT(priv->hw_vport != NULL);
if (priv->hw_vport[v->port_id]) {
acts->rule_acts[action_dst].action =
priv->hw_vport[v->port_id];
} else {
return rte_flow_error_set
(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"cannot use represented_port action"
" with this port");
}
} else {
ret = __flow_hw_act_data_general_append
(priv, acts, action->type,
action - action_start, action_dst);
if (ret)
return rte_flow_error_set
(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"not enough memory to store"
" vport action");
}
return 0;
}
static __rte_always_inline int
flow_hw_meter_compile(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
uint16_t aso_mtr_pos,
uint16_t jump_pos,
const struct rte_flow_action *action,
struct mlx5_hw_actions *acts,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr *aso_mtr;
const struct rte_flow_action_meter *meter = action->conf;
uint32_t group = cfg->attr.flow_attr.group;
aso_mtr = mlx5_aso_meter_by_idx(priv, meter->mtr_id);
acts->rule_acts[aso_mtr_pos].action = priv->mtr_bulk.action;
acts->rule_acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
acts->jump = flow_hw_jump_action_register
(dev, cfg, aso_mtr->fm.group, error);
if (!acts->jump)
return -ENOMEM;
acts->rule_acts[jump_pos].action = (!!group) ?
acts->jump->hws_action :
acts->jump->root_action;
if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
return -ENOMEM;
return 0;
}
static __rte_always_inline int
flow_hw_cnt_compile(struct rte_eth_dev *dev, uint32_t start_pos,
struct mlx5_hw_actions *acts)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t pos = start_pos;
cnt_id_t cnt_id;
int ret;
ret = mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0);
if (ret != 0)
return ret;
ret = mlx5_hws_cnt_pool_get_action_offset
(priv->hws_cpool,
cnt_id,
&acts->rule_acts[pos].action,
&acts->rule_acts[pos].counter.offset);
if (ret != 0)
return ret;
acts->cnt_id = cnt_id;
return 0;
}
static __rte_always_inline bool
is_of_vlan_pcp_present(const struct rte_flow_action *actions)
{
/*
* Order of RTE VLAN push actions is
* OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
*/
return actions[MLX5_HW_VLAN_PUSH_PCP_IDX].type ==
RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP;
}
static __rte_always_inline bool
is_template_masked_push_vlan(const struct rte_flow_action_of_push_vlan *mask)
{
/*
* In masked push VLAN template all RTE push actions are masked.
*/
return mask && mask->ethertype != 0;
}
static rte_be32_t vlan_hdr_to_be32(const struct rte_flow_action *actions)
{
/*
* OpenFlow Switch Specification defines 801.1q VID as 12+1 bits.
*/
rte_be32_t type, vid, pcp;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
rte_be32_t vid_lo, vid_hi;
#endif
type = ((const struct rte_flow_action_of_push_vlan *)
actions[MLX5_HW_VLAN_PUSH_TYPE_IDX].conf)->ethertype;
vid = ((const struct rte_flow_action_of_set_vlan_vid *)
actions[MLX5_HW_VLAN_PUSH_VID_IDX].conf)->vlan_vid;
pcp = is_of_vlan_pcp_present(actions) ?
((const struct rte_flow_action_of_set_vlan_pcp *)
actions[MLX5_HW_VLAN_PUSH_PCP_IDX].conf)->vlan_pcp : 0;
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
vid_hi = vid & 0xff;
vid_lo = vid >> 8;
return (((vid_lo << 8) | (pcp << 5) | vid_hi) << 16) | type;
#else
return (type << 16) | (pcp << 13) | vid;
#endif
}
static __rte_always_inline struct mlx5_aso_mtr *
flow_hw_meter_mark_alloc(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_action *action,
void *user_data, bool push)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
const struct rte_flow_action_meter_mark *meter_mark = action->conf;
struct mlx5_aso_mtr *aso_mtr;
struct mlx5_flow_meter_info *fm;
uint32_t mtr_id;
aso_mtr = mlx5_ipool_malloc(priv->hws_mpool->idx_pool, &mtr_id);
if (!aso_mtr)
return NULL;
/* Fill the flow meter parameters. */
aso_mtr->type = ASO_METER_INDIRECT;
fm = &aso_mtr->fm;
fm->meter_id = mtr_id;
fm->profile = (struct mlx5_flow_meter_profile *)(meter_mark->profile);
fm->is_enable = meter_mark->state;
fm->color_aware = meter_mark->color_mode;
aso_mtr->pool = pool;
aso_mtr->state = (queue == MLX5_HW_INV_QUEUE) ?
ASO_METER_WAIT : ASO_METER_WAIT_ASYNC;
aso_mtr->offset = mtr_id - 1;
aso_mtr->init_color = (meter_mark->color_mode) ?
meter_mark->init_color : RTE_COLOR_GREEN;
/* Update ASO flow meter by wqe. */
if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
&priv->mtr_bulk, user_data, push)) {
mlx5_ipool_free(pool->idx_pool, mtr_id);
return NULL;
}
/* Wait for ASO object completion. */
if (queue == MLX5_HW_INV_QUEUE &&
mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
mlx5_ipool_free(pool->idx_pool, mtr_id);
return NULL;
}
return aso_mtr;
}
static __rte_always_inline int
flow_hw_meter_mark_compile(struct rte_eth_dev *dev,
uint16_t aso_mtr_pos,
const struct rte_flow_action *action,
struct mlx5dr_rule_action *acts,
uint32_t *index,
uint32_t queue)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
struct mlx5_aso_mtr *aso_mtr;
aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, NULL, true);
if (!aso_mtr)
return -1;
/* Compile METER_MARK action */
acts[aso_mtr_pos].action = pool->action;
acts[aso_mtr_pos].aso_meter.offset = aso_mtr->offset;
acts[aso_mtr_pos].aso_meter.init_color =
(enum mlx5dr_action_aso_meter_color)
rte_col_2_mlx5_col(aso_mtr->init_color);
*index = aso_mtr->fm.meter_id;
return 0;
}
/**
* Translate rte_flow actions to DR action.
*
* As the action template has already indicated the actions. Translate
* the rte_flow actions to DR action if possbile. So in flow create
* stage we will save cycles from handing the actions' organizing.
* For the actions with limited information, need to add these to a
* list.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] cfg
* Pointer to the table configuration.
* @param[in/out] acts
* Pointer to the template HW steering DR actions.
* @param[in] at
* Action template.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno otherwise and rte_errno is set.
*/
static int
__flow_hw_actions_translate(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
struct mlx5_hw_actions *acts,
struct rte_flow_actions_template *at,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_template_table_attr *table_attr = &cfg->attr;
const struct rte_flow_attr *attr = &table_attr->flow_attr;
struct rte_flow_action *actions = at->actions;
struct rte_flow_action *action_start = actions;
struct rte_flow_action *masks = at->masks;
enum mlx5dr_action_reformat_type refmt_type = 0;
const struct rte_flow_action_raw_encap *raw_encap_data;
const struct rte_flow_item *enc_item = NULL, *enc_item_m = NULL;
uint16_t reformat_src = 0;
uint8_t *encap_data = NULL, *encap_data_m = NULL;
size_t data_size = 0;
struct mlx5_hw_modify_header_action mhdr = { 0 };
bool actions_end = false;
uint32_t type;
bool reformat_used = false;
uint16_t action_pos;
uint16_t jump_pos;
uint32_t ct_idx;
int err;
uint32_t target_grp = 0;
flow_hw_modify_field_init(&mhdr, at);
if (attr->transfer)
type = MLX5DR_TABLE_TYPE_FDB;
else if (attr->egress)
type = MLX5DR_TABLE_TYPE_NIC_TX;
else
type = MLX5DR_TABLE_TYPE_NIC_RX;
for (; !actions_end; actions++, masks++) {
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_INDIRECT:
action_pos = at->actions_off[actions - at->actions];
if (!attr->group) {
DRV_LOG(ERR, "Indirect action is not supported in root table.");
goto err;
}
if (actions->conf && masks->conf) {
if (flow_hw_shared_action_translate
(dev, actions, acts, actions - action_start, action_pos))
goto err;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)){
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_VOID:
break;
case RTE_FLOW_ACTION_TYPE_DROP:
action_pos = at->actions_off[actions - at->actions];
acts->rule_acts[action_pos].action =
priv->hw_drop[!!attr->group];
break;
case RTE_FLOW_ACTION_TYPE_MARK:
action_pos = at->actions_off[actions - at->actions];
acts->mark = true;
if (masks->conf &&
((const struct rte_flow_action_mark *)
masks->conf)->id)
acts->rule_acts[action_pos].tag.value =
mlx5_flow_mark_set
(((const struct rte_flow_action_mark *)
(actions->conf))->id);
else if (__flow_hw_act_data_general_append(priv, acts,
actions->type, actions - action_start, action_pos))
goto err;
acts->rule_acts[action_pos].action =
priv->hw_tag[!!attr->group];
flow_hw_rxq_flag_set(dev, true);
break;
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
action_pos = at->actions_off[actions - at->actions];
acts->rule_acts[action_pos].action =
priv->hw_push_vlan[type];
if (is_template_masked_push_vlan(masks->conf))
acts->rule_acts[action_pos].push_vlan.vlan_hdr =
vlan_hdr_to_be32(actions);
else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos))
goto err;
actions += is_of_vlan_pcp_present(actions) ?
MLX5_HW_VLAN_PUSH_PCP_IDX :
MLX5_HW_VLAN_PUSH_VID_IDX;
break;
case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
action_pos = at->actions_off[actions - at->actions];
acts->rule_acts[action_pos].action =
priv->hw_pop_vlan[type];
break;
case RTE_FLOW_ACTION_TYPE_JUMP:
action_pos = at->actions_off[actions - at->actions];
if (masks->conf &&
((const struct rte_flow_action_jump *)
masks->conf)->group) {
uint32_t jump_group =
((const struct rte_flow_action_jump *)
actions->conf)->group;
acts->jump = flow_hw_jump_action_register
(dev, cfg, jump_group, error);
if (!acts->jump)
goto err;
acts->rule_acts[action_pos].action = (!!attr->group) ?
acts->jump->hws_action :
acts->jump->root_action;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)){
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
action_pos = at->actions_off[actions - at->actions];
if (masks->conf &&
((const struct rte_flow_action_queue *)
masks->conf)->index) {
acts->tir = flow_hw_tir_action_register
(dev,
mlx5_hw_act_flag[!!attr->group][type],
actions);
if (!acts->tir)
goto err;
acts->rule_acts[action_pos].action =
acts->tir->action;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)) {
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_RSS:
action_pos = at->actions_off[actions - at->actions];
if (actions->conf && masks->conf) {
acts->tir = flow_hw_tir_action_register
(dev,
mlx5_hw_act_flag[!!attr->group][type],
actions);
if (!acts->tir)
goto err;
acts->rule_acts[action_pos].action =
acts->tir->action;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)) {
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
MLX5_ASSERT(!reformat_used);
enc_item = ((const struct rte_flow_action_vxlan_encap *)
actions->conf)->definition;
if (masks->conf)
enc_item_m = ((const struct rte_flow_action_vxlan_encap *)
masks->conf)->definition;
reformat_used = true;
reformat_src = actions - action_start;
refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
break;
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
MLX5_ASSERT(!reformat_used);
enc_item = ((const struct rte_flow_action_nvgre_encap *)
actions->conf)->definition;
if (masks->conf)
enc_item_m = ((const struct rte_flow_action_nvgre_encap *)
masks->conf)->definition;
reformat_used = true;
reformat_src = actions - action_start;
refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
MLX5_ASSERT(!reformat_used);
reformat_used = true;
refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
raw_encap_data =
(const struct rte_flow_action_raw_encap *)
masks->conf;
if (raw_encap_data)
encap_data_m = raw_encap_data->data;
raw_encap_data =
(const struct rte_flow_action_raw_encap *)
actions->conf;
encap_data = raw_encap_data->data;
data_size = raw_encap_data->size;
if (reformat_used) {
refmt_type = data_size <
MLX5_ENCAPSULATION_DECISION_SIZE ?
MLX5DR_ACTION_REFORMAT_TYPE_TNL_L3_TO_L2 :
MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L3;
} else {
reformat_used = true;
refmt_type =
MLX5DR_ACTION_REFORMAT_TYPE_L2_TO_TNL_L2;
}
reformat_src = actions - action_start;
break;
case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
reformat_used = true;
refmt_type = MLX5DR_ACTION_REFORMAT_TYPE_TNL_L2_TO_L2;
break;
case RTE_FLOW_ACTION_TYPE_SEND_TO_KERNEL:
DRV_LOG(ERR, "send to kernel action is not supported in HW steering.");
goto err;
case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
err = flow_hw_modify_field_compile(dev, attr, action_start,
actions, masks, acts, &mhdr,
error);
if (err)
goto err;
/*
* Adjust the action source position for the following.
* ... / MODIFY_FIELD: rx_cpy_pos / (QUEUE|RSS) / ...
* The next action will be Q/RSS, there will not be
* another adjustment and the real source position of
* the following actions will be decreased by 1.
* No change of the total actions in the new template.
*/
if ((actions - action_start) == at->rx_cpy_pos)
action_start += 1;
break;
case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
action_pos = at->actions_off[actions - at->actions];
if (flow_hw_represented_port_compile
(dev, attr, action_start, actions,
masks, acts, action_pos, error))
goto err;
break;
case RTE_FLOW_ACTION_TYPE_METER:
/*
* METER action is compiled to 2 DR actions - ASO_METER and FT.
* Calculated DR offset is stored only for ASO_METER and FT
* is assumed to be the next action.
*/
action_pos = at->actions_off[actions - at->actions];
jump_pos = action_pos + 1;
if (actions->conf && masks->conf &&
((const struct rte_flow_action_meter *)
masks->conf)->mtr_id) {
err = flow_hw_meter_compile(dev, cfg,
action_pos, jump_pos, actions, acts, error);
if (err)
goto err;
} else if (__flow_hw_act_data_general_append(priv, acts,
actions->type,
actions - action_start,
action_pos))
goto err;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
flow_hw_translate_group(dev, cfg, attr->group,
&target_grp, error);
if (target_grp == 0) {
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"Age action on root table is not supported in HW steering mode");
}
action_pos = at->actions_off[actions - at->actions];
if (__flow_hw_act_data_general_append(priv, acts,
actions->type,
actions - action_start,
action_pos))
goto err;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
flow_hw_translate_group(dev, cfg, attr->group,
&target_grp, error);
if (target_grp == 0) {
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"Counter action on root table is not supported in HW steering mode");
}
if ((at->action_flags & MLX5_FLOW_ACTION_AGE) ||
(at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
/*
* When both COUNT and AGE are requested, it is
* saved as AGE action which creates also the
* counter.
*/
break;
action_pos = at->actions_off[actions - at->actions];
if (masks->conf &&
((const struct rte_flow_action_count *)
masks->conf)->id) {
err = flow_hw_cnt_compile(dev, action_pos, acts);
if (err)
goto err;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)) {
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
action_pos = at->actions_off[actions - at->actions];
if (masks->conf) {
ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
((uint32_t)(uintptr_t)actions->conf);
if (flow_hw_ct_compile(dev, MLX5_HW_INV_QUEUE, ct_idx,
&acts->rule_acts[action_pos]))
goto err;
} else if (__flow_hw_act_data_general_append
(priv, acts, actions->type,
actions - action_start, action_pos)) {
goto err;
}
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
action_pos = at->actions_off[actions - at->actions];
if (actions->conf && masks->conf &&
((const struct rte_flow_action_meter_mark *)
masks->conf)->profile) {
err = flow_hw_meter_mark_compile(dev,
action_pos, actions,
acts->rule_acts,
&acts->mtr_id,
MLX5_HW_INV_QUEUE);
if (err)
goto err;
} else if (__flow_hw_act_data_general_append(priv, acts,
actions->type,
actions - action_start,
action_pos))
goto err;
break;
case RTE_FLOW_ACTION_TYPE_END:
actions_end = true;
break;
default:
break;
}
}
if (mhdr.pos != UINT16_MAX) {
uint32_t flags;
uint32_t bulk_size;
size_t mhdr_len;
acts->mhdr = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*acts->mhdr),
0, SOCKET_ID_ANY);
if (!acts->mhdr)
goto err;
rte_memcpy(acts->mhdr, &mhdr, sizeof(*acts->mhdr));
mhdr_len = sizeof(struct mlx5_modification_cmd) * acts->mhdr->mhdr_cmds_num;
flags = mlx5_hw_act_flag[!!attr->group][type];
if (acts->mhdr->shared) {
flags |= MLX5DR_ACTION_FLAG_SHARED;
bulk_size = 0;
} else {
bulk_size = rte_log2_u32(table_attr->nb_flows);
}
acts->mhdr->action = mlx5dr_action_create_modify_header
(priv->dr_ctx, mhdr_len, (__be64 *)acts->mhdr->mhdr_cmds,
bulk_size, flags);
if (!acts->mhdr->action)
goto err;
acts->rule_acts[acts->mhdr->pos].action = acts->mhdr->action;
}
if (reformat_used) {
uint8_t buf[MLX5_ENCAP_MAX_LEN];
bool shared_rfmt = true;
MLX5_ASSERT(at->reformat_off != UINT16_MAX);
if (enc_item) {
MLX5_ASSERT(!encap_data);
if (flow_dv_convert_encap_data(enc_item, buf, &data_size, error))
goto err;
encap_data = buf;
if (!enc_item_m)
shared_rfmt = false;
} else if (encap_data && !encap_data_m) {
shared_rfmt = false;
}
acts->encap_decap = mlx5_malloc(MLX5_MEM_ZERO,
sizeof(*acts->encap_decap) + data_size,
0, SOCKET_ID_ANY);
if (!acts->encap_decap)
goto err;
if (data_size) {
acts->encap_decap->data_size = data_size;
memcpy(acts->encap_decap->data, encap_data, data_size);
}
acts->encap_decap->action = mlx5dr_action_create_reformat
(priv->dr_ctx, refmt_type,
data_size, encap_data,
shared_rfmt ? 0 : rte_log2_u32(table_attr->nb_flows),
mlx5_hw_act_flag[!!attr->group][type] |
(shared_rfmt ? MLX5DR_ACTION_FLAG_SHARED : 0));
if (!acts->encap_decap->action)
goto err;
acts->rule_acts[at->reformat_off].action = acts->encap_decap->action;
acts->rule_acts[at->reformat_off].reformat.data = acts->encap_decap->data;
if (shared_rfmt)
acts->rule_acts[at->reformat_off].reformat.offset = 0;
else if (__flow_hw_act_data_encap_append(priv, acts,
(action_start + reformat_src)->type,
reformat_src, at->reformat_off, data_size))
goto err;
acts->encap_decap->shared = shared_rfmt;
acts->encap_decap_pos = at->reformat_off;
}
return 0;
err:
err = rte_errno;
__flow_hw_action_template_destroy(dev, acts);
return rte_flow_error_set(error, err,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to create rte table");
}
/**
* Translate rte_flow actions to DR action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] tbl
* Pointer to the flow template table.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_actions_translate(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
struct rte_flow_error *error)
{
uint32_t i;
for (i = 0; i < tbl->nb_action_templates; i++) {
if (__flow_hw_actions_translate(dev, &tbl->cfg,
&tbl->ats[i].acts,
tbl->ats[i].action_template,
error))
goto err;
}
return 0;
err:
while (i--)
__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
return -1;
}
/**
* Get shared indirect action.
*
* @param[in] dev
* Pointer to the rte_eth_dev data structure.
* @param[in] act_data
* Pointer to the recorded action construct data.
* @param[in] item_flags
* The matcher itme_flags used for RSS lookup.
* @param[in] rule_act
* Pointer to the shared action's destination rule DR action.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
flow_hw_shared_action_get(struct rte_eth_dev *dev,
struct mlx5_action_construct_data *act_data,
const uint64_t item_flags,
struct mlx5dr_rule_action *rule_act)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_rss_desc rss_desc = { 0 };
uint64_t hash_fields = 0;
uint32_t hrxq_idx = 0;
struct mlx5_hrxq *hrxq = NULL;
int act_type = act_data->type;
switch (act_type) {
case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
rss_desc.level = act_data->shared_rss.level;
rss_desc.types = act_data->shared_rss.types;
flow_dv_hashfields_set(item_flags, &rss_desc, &hash_fields);
hrxq_idx = flow_dv_action_rss_hrxq_lookup
(dev, act_data->shared_rss.idx, hash_fields);
if (hrxq_idx)
hrxq = mlx5_ipool_get(priv->sh->ipool[MLX5_IPOOL_HRXQ],
hrxq_idx);
if (hrxq) {
rule_act->action = hrxq->action;
return 0;
}
break;
default:
DRV_LOG(WARNING, "Unsupported shared action type:%d",
act_data->type);
break;
}
return -1;
}
/**
* Construct shared indirect action.
*
* @param[in] dev
* Pointer to the rte_eth_dev data structure.
* @param[in] queue
* The flow creation queue index.
* @param[in] action
* Pointer to the shared indirect rte_flow action.
* @param[in] table
* Pointer to the flow table.
* @param[in] it_idx
* Item template index the action template refer to.
* @param[in] action_flags
* Actions bit-map detected in this template.
* @param[in, out] flow
* Pointer to the flow containing the counter.
* @param[in] rule_act
* Pointer to the shared action's destination rule DR action.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
flow_hw_shared_action_construct(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_action *action,
struct rte_flow_template_table *table,
const uint8_t it_idx, uint64_t action_flags,
struct rte_flow_hw *flow,
struct mlx5dr_rule_action *rule_act)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
struct mlx5_action_construct_data act_data;
struct mlx5_shared_action_rss *shared_rss;
struct mlx5_aso_mtr *aso_mtr;
struct mlx5_age_info *age_info;
struct mlx5_hws_age_param *param;
uint32_t act_idx = (uint32_t)(uintptr_t)action->conf;
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t idx = act_idx &
((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
uint64_t item_flags;
cnt_id_t age_cnt;
memset(&act_data, 0, sizeof(act_data));
switch (type) {
case MLX5_INDIRECT_ACTION_TYPE_RSS:
act_data.type = MLX5_RTE_FLOW_ACTION_TYPE_RSS;
shared_rss = mlx5_ipool_get
(priv->sh->ipool[MLX5_IPOOL_RSS_SHARED_ACTIONS], idx);
if (!shared_rss)
return -1;
act_data.shared_rss.idx = idx;
act_data.shared_rss.level = shared_rss->origin.level;
act_data.shared_rss.types = !shared_rss->origin.types ?
RTE_ETH_RSS_IP :
shared_rss->origin.types;
item_flags = table->its[it_idx]->item_flags;
if (flow_hw_shared_action_get
(dev, &act_data, item_flags, rule_act))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_COUNT:
if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
act_idx,
&rule_act->action,
&rule_act->counter.offset))
return -1;
flow->cnt_id = act_idx;
break;
case MLX5_INDIRECT_ACTION_TYPE_AGE:
/*
* Save the index with the indirect type, to recognize
* it in flow destroy.
*/
flow->age_idx = act_idx;
if (action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
/*
* The mutual update for idirect AGE & COUNT will be
* performed later after we have ID for both of them.
*/
break;
age_info = GET_PORT_AGE_INFO(priv);
param = mlx5_ipool_get(age_info->ages_ipool, idx);
if (param == NULL)
return -1;
if (action_flags & MLX5_FLOW_ACTION_COUNT) {
if (mlx5_hws_cnt_pool_get(priv->hws_cpool,
&param->queue_id, &age_cnt,
idx) < 0)
return -1;
flow->cnt_id = age_cnt;
param->nb_cnts++;
} else {
/*
* Get the counter of this indirect AGE or create one
* if doesn't exist.
*/
age_cnt = mlx5_hws_age_cnt_get(priv, param, idx);
if (age_cnt == 0)
return -1;
}
if (mlx5_hws_cnt_pool_get_action_offset(priv->hws_cpool,
age_cnt, &rule_act->action,
&rule_act->counter.offset))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_CT:
if (flow_hw_ct_compile(dev, queue, idx, rule_act))
return -1;
break;
case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
/* Find ASO object. */
aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
if (!aso_mtr)
return -1;
rule_act->action = pool->action;
rule_act->aso_meter.offset = aso_mtr->offset;
rule_act->aso_meter.init_color =
(enum mlx5dr_action_aso_meter_color)
rte_col_2_mlx5_col(aso_mtr->init_color);
break;
default:
DRV_LOG(WARNING, "Unsupported shared action type:%d", type);
break;
}
return 0;
}
static __rte_always_inline int
flow_hw_mhdr_cmd_is_nop(const struct mlx5_modification_cmd *cmd)
{
struct mlx5_modification_cmd cmd_he = {
.data0 = rte_be_to_cpu_32(cmd->data0),
.data1 = 0,
};
return cmd_he.action_type == MLX5_MODIFICATION_TYPE_NOP;
}
/**
* Construct flow action array.
*
* For action template contains dynamic actions, these actions need to
* be updated according to the rte_flow action during flow creation.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] job
* Pointer to job descriptor.
* @param[in] hw_acts
* Pointer to translated actions from template.
* @param[in] it_idx
* Item template index the action template refer to.
* @param[in] actions
* Array of rte_flow action need to be checked.
* @param[in] rule_acts
* Array of DR rule actions to be used during flow creation..
* @param[in] acts_num
* Pointer to the real acts_num flow has.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
flow_hw_modify_field_construct(struct mlx5_hw_q_job *job,
struct mlx5_action_construct_data *act_data,
const struct mlx5_hw_actions *hw_acts,
const struct rte_flow_action *action)
{
const struct rte_flow_action_modify_field *mhdr_action = action->conf;
uint8_t values[16] = { 0 };
unaligned_uint32_t *value_p;
uint32_t i;
struct field_modify_info *field;
if (!hw_acts->mhdr)
return -1;
if (hw_acts->mhdr->shared || act_data->modify_header.shared)
return 0;
MLX5_ASSERT(mhdr_action->operation == RTE_FLOW_MODIFY_SET ||
mhdr_action->operation == RTE_FLOW_MODIFY_ADD);
if (mhdr_action->src.field != RTE_FLOW_FIELD_VALUE &&
mhdr_action->src.field != RTE_FLOW_FIELD_POINTER)
return 0;
if (mhdr_action->src.field == RTE_FLOW_FIELD_VALUE)
rte_memcpy(values, &mhdr_action->src.value, sizeof(values));
else
rte_memcpy(values, mhdr_action->src.pvalue, sizeof(values));
if (mhdr_action->dst.field == RTE_FLOW_FIELD_META ||
mhdr_action->dst.field == RTE_FLOW_FIELD_TAG ||
mhdr_action->dst.field == RTE_FLOW_FIELD_METER_COLOR ||
mhdr_action->dst.field == (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG) {
value_p = (unaligned_uint32_t *)values;
*value_p = rte_cpu_to_be_32(*value_p);
} else if (mhdr_action->dst.field == RTE_FLOW_FIELD_GTP_PSC_QFI) {
uint32_t tmp;
/*
* QFI is passed as an uint8_t integer, but it is accessed through
* a 2nd least significant byte of a 32-bit field in modify header command.
*/
tmp = values[0];
value_p = (unaligned_uint32_t *)values;
*value_p = rte_cpu_to_be_32(tmp << 8);
}
i = act_data->modify_header.mhdr_cmds_off;
field = act_data->modify_header.field;
do {
uint32_t off_b;
uint32_t mask;
uint32_t data;
const uint8_t *mask_src;
if (i >= act_data->modify_header.mhdr_cmds_end)
return -1;
if (flow_hw_mhdr_cmd_is_nop(&job->mhdr_cmd[i])) {
++i;
continue;
}
mask_src = (const uint8_t *)act_data->modify_header.mask;
mask = flow_dv_fetch_field(mask_src + field->offset, field->size);
if (!mask) {
++field;
continue;
}
off_b = rte_bsf32(mask);
data = flow_dv_fetch_field(values + field->offset, field->size);
data = (data & mask) >> off_b;
job->mhdr_cmd[i++].data1 = rte_cpu_to_be_32(data);
++field;
} while (field->size);
return 0;
}
/**
* Construct flow action array.
*
* For action template contains dynamic actions, these actions need to
* be updated according to the rte_flow action during flow creation.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] job
* Pointer to job descriptor.
* @param[in] hw_acts
* Pointer to translated actions from template.
* @param[in] it_idx
* Item template index the action template refer to.
* @param[in] actions
* Array of rte_flow action need to be checked.
* @param[in] rule_acts
* Array of DR rule actions to be used during flow creation..
* @param[in] acts_num
* Pointer to the real acts_num flow has.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static __rte_always_inline int
flow_hw_actions_construct(struct rte_eth_dev *dev,
struct mlx5_hw_q_job *job,
const struct mlx5_hw_action_template *hw_at,
const uint8_t it_idx,
const struct rte_flow_action actions[],
struct mlx5dr_rule_action *rule_acts,
uint32_t queue,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
struct rte_flow_template_table *table = job->flow->table;
struct mlx5_action_construct_data *act_data;
const struct rte_flow_actions_template *at = hw_at->action_template;
const struct mlx5_hw_actions *hw_acts = &hw_at->acts;
const struct rte_flow_action *action;
const struct rte_flow_action_raw_encap *raw_encap_data;
const struct rte_flow_item *enc_item = NULL;
const struct rte_flow_action_ethdev *port_action = NULL;
const struct rte_flow_action_meter *meter = NULL;
const struct rte_flow_action_age *age = NULL;
uint8_t *buf = job->encap_data;
struct rte_flow_attr attr = {
.ingress = 1,
};
uint32_t ft_flag;
size_t encap_len = 0;
int ret;
uint32_t age_idx = 0;
struct mlx5_aso_mtr *aso_mtr;
rte_memcpy(rule_acts, hw_acts->rule_acts, sizeof(*rule_acts) * at->dr_actions_num);
attr.group = table->grp->group_id;
ft_flag = mlx5_hw_act_flag[!!table->grp->group_id][table->type];
if (table->type == MLX5DR_TABLE_TYPE_FDB) {
attr.transfer = 1;
attr.ingress = 1;
} else if (table->type == MLX5DR_TABLE_TYPE_NIC_TX) {
attr.egress = 1;
attr.ingress = 0;
} else {
attr.ingress = 1;
}
if (hw_acts->mhdr && hw_acts->mhdr->mhdr_cmds_num > 0) {
uint16_t pos = hw_acts->mhdr->pos;
if (!hw_acts->mhdr->shared) {
rule_acts[pos].modify_header.offset =
job->flow->idx - 1;
rule_acts[pos].modify_header.data =
(uint8_t *)job->mhdr_cmd;
rte_memcpy(job->mhdr_cmd, hw_acts->mhdr->mhdr_cmds,
sizeof(*job->mhdr_cmd) * hw_acts->mhdr->mhdr_cmds_num);
}
}
LIST_FOREACH(act_data, &hw_acts->act_list, next) {
uint32_t jump_group;
uint32_t tag;
uint64_t item_flags;
struct mlx5_hw_jump_action *jump;
struct mlx5_hrxq *hrxq;
uint32_t ct_idx;
cnt_id_t cnt_id;
uint32_t mtr_id;
action = &actions[act_data->action_src];
/*
* action template construction replaces
* OF_SET_VLAN_VID with MODIFY_FIELD
*/
if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
MLX5_ASSERT(act_data->type ==
RTE_FLOW_ACTION_TYPE_MODIFY_FIELD);
else
MLX5_ASSERT(action->type ==
RTE_FLOW_ACTION_TYPE_INDIRECT ||
(int)action->type == act_data->type);
switch (act_data->type) {
case RTE_FLOW_ACTION_TYPE_INDIRECT:
if (flow_hw_shared_action_construct
(dev, queue, action, table, it_idx,
at->action_flags, job->flow,
&rule_acts[act_data->action_dst]))
return -1;
break;
case RTE_FLOW_ACTION_TYPE_VOID:
break;
case RTE_FLOW_ACTION_TYPE_MARK:
tag = mlx5_flow_mark_set
(((const struct rte_flow_action_mark *)
(action->conf))->id);
rule_acts[act_data->action_dst].tag.value = tag;
break;
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
rule_acts[act_data->action_dst].push_vlan.vlan_hdr =
vlan_hdr_to_be32(action);
break;
case RTE_FLOW_ACTION_TYPE_JUMP:
jump_group = ((const struct rte_flow_action_jump *)
action->conf)->group;
jump = flow_hw_jump_action_register
(dev, &table->cfg, jump_group, NULL);
if (!jump)
return -1;
rule_acts[act_data->action_dst].action =
(!!attr.group) ? jump->hws_action : jump->root_action;
job->flow->jump = jump;
job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
case RTE_FLOW_ACTION_TYPE_QUEUE:
hrxq = flow_hw_tir_action_register(dev,
ft_flag,
action);
if (!hrxq)
return -1;
rule_acts[act_data->action_dst].action = hrxq->action;
job->flow->hrxq = hrxq;
job->flow->fate_type = MLX5_FLOW_FATE_QUEUE;
break;
case MLX5_RTE_FLOW_ACTION_TYPE_RSS:
item_flags = table->its[it_idx]->item_flags;
if (flow_hw_shared_action_get
(dev, act_data, item_flags,
&rule_acts[act_data->action_dst]))
return -1;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
enc_item = ((const struct rte_flow_action_vxlan_encap *)
action->conf)->definition;
if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
return -1;
break;
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
enc_item = ((const struct rte_flow_action_nvgre_encap *)
action->conf)->definition;
if (flow_dv_convert_encap_data(enc_item, buf, &encap_len, NULL))
return -1;
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
raw_encap_data =
(const struct rte_flow_action_raw_encap *)
action->conf;
rte_memcpy((void *)buf, raw_encap_data->data, act_data->encap.len);
MLX5_ASSERT(raw_encap_data->size ==
act_data->encap.len);
break;
case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
if (action->type == RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
ret = flow_hw_set_vlan_vid_construct(dev, job,
act_data,
hw_acts,
action);
else
ret = flow_hw_modify_field_construct(job,
act_data,
hw_acts,
action);
if (ret)
return -1;
break;
case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
port_action = action->conf;
if (!priv->hw_vport[port_action->port_id])
return -1;
rule_acts[act_data->action_dst].action =
priv->hw_vport[port_action->port_id];
break;
case RTE_FLOW_ACTION_TYPE_METER:
meter = action->conf;
mtr_id = meter->mtr_id;
aso_mtr = mlx5_aso_meter_by_idx(priv, mtr_id);
rule_acts[act_data->action_dst].action =
priv->mtr_bulk.action;
rule_acts[act_data->action_dst].aso_meter.offset =
aso_mtr->offset;
jump = flow_hw_jump_action_register
(dev, &table->cfg, aso_mtr->fm.group, NULL);
if (!jump)
return -1;
MLX5_ASSERT
(!rule_acts[act_data->action_dst + 1].action);
rule_acts[act_data->action_dst + 1].action =
(!!attr.group) ? jump->hws_action :
jump->root_action;
job->flow->jump = jump;
job->flow->fate_type = MLX5_FLOW_FATE_JUMP;
if (mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr))
return -1;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
age = action->conf;
/*
* First, create the AGE parameter, then create its
* counter later:
* Regular counter - in next case.
* Indirect counter - update it after the loop.
*/
age_idx = mlx5_hws_age_action_create(priv, queue, 0,
age,
job->flow->idx,
error);
if (age_idx == 0)
return -rte_errno;
job->flow->age_idx = age_idx;
if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT)
/*
* When AGE uses indirect counter, no need to
* create counter but need to update it with the
* AGE parameter, will be done after the loop.
*/
break;
/* Fall-through. */
case RTE_FLOW_ACTION_TYPE_COUNT:
ret = mlx5_hws_cnt_pool_get(priv->hws_cpool, &queue,
&cnt_id, age_idx);
if (ret != 0)
return ret;
ret = mlx5_hws_cnt_pool_get_action_offset
(priv->hws_cpool,
cnt_id,
&rule_acts[act_data->action_dst].action,
&rule_acts[act_data->action_dst].counter.offset
);
if (ret != 0)
return ret;
job->flow->cnt_id = cnt_id;
break;
case MLX5_RTE_FLOW_ACTION_TYPE_COUNT:
ret = mlx5_hws_cnt_pool_get_action_offset
(priv->hws_cpool,
act_data->shared_counter.id,
&rule_acts[act_data->action_dst].action,
&rule_acts[act_data->action_dst].counter.offset
);
if (ret != 0)
return ret;
job->flow->cnt_id = act_data->shared_counter.id;
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
ct_idx = MLX5_ACTION_CTX_CT_GET_IDX
((uint32_t)(uintptr_t)action->conf);
if (flow_hw_ct_compile(dev, queue, ct_idx,
&rule_acts[act_data->action_dst]))
return -1;
break;
case MLX5_RTE_FLOW_ACTION_TYPE_METER_MARK:
mtr_id = act_data->shared_meter.id &
((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
/* Find ASO object. */
aso_mtr = mlx5_ipool_get(pool->idx_pool, mtr_id);
if (!aso_mtr)
return -1;
rule_acts[act_data->action_dst].action =
pool->action;
rule_acts[act_data->action_dst].aso_meter.offset =
aso_mtr->offset;
rule_acts[act_data->action_dst].aso_meter.init_color =
(enum mlx5dr_action_aso_meter_color)
rte_col_2_mlx5_col(aso_mtr->init_color);
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
/*
* Allocate meter directly will slow down flow
* insertion rate.
*/
ret = flow_hw_meter_mark_compile(dev,
act_data->action_dst, action,
rule_acts, &job->flow->mtr_id, MLX5_HW_INV_QUEUE);
if (ret != 0)
return ret;
break;
default:
break;
}
}
if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT) {
if (at->action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE) {
age_idx = job->flow->age_idx & MLX5_HWS_AGE_IDX_MASK;
if (mlx5_hws_cnt_age_get(priv->hws_cpool,
job->flow->cnt_id) != age_idx)
/*
* This is first use of this indirect counter
* for this indirect AGE, need to increase the
* number of counters.
*/
mlx5_hws_age_nb_cnt_increase(priv, age_idx);
}
/*
* Update this indirect counter the indirect/direct AGE in which
* using it.
*/
mlx5_hws_cnt_age_set(priv->hws_cpool, job->flow->cnt_id,
age_idx);
}
if (hw_acts->encap_decap && !hw_acts->encap_decap->shared) {
rule_acts[hw_acts->encap_decap_pos].reformat.offset =
job->flow->idx - 1;
rule_acts[hw_acts->encap_decap_pos].reformat.data = buf;
}
if (mlx5_hws_cnt_id_valid(hw_acts->cnt_id))
job->flow->cnt_id = hw_acts->cnt_id;
return 0;
}
static const struct rte_flow_item *
flow_hw_get_rule_items(struct rte_eth_dev *dev,
struct rte_flow_template_table *table,
const struct rte_flow_item items[],
uint8_t pattern_template_index,
struct mlx5_hw_q_job *job)
{
struct rte_flow_pattern_template *pt = table->its[pattern_template_index];
/* Only one implicit item can be added to flow rule pattern. */
MLX5_ASSERT(!pt->implicit_port || !pt->implicit_tag);
/* At least one item was allocated in job descriptor for items. */
MLX5_ASSERT(MLX5_HW_MAX_ITEMS >= 1);
if (pt->implicit_port) {
if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
rte_errno = ENOMEM;
return NULL;
}
/* Set up represented port item in job descriptor. */
job->port_spec = (struct rte_flow_item_ethdev){
.port_id = dev->data->port_id,
};
job->items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.spec = &job->port_spec,
};
rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
return job->items;
} else if (pt->implicit_tag) {
if (pt->orig_item_nb + 1 > MLX5_HW_MAX_ITEMS) {
rte_errno = ENOMEM;
return NULL;
}
/* Set up tag item in job descriptor. */
job->tag_spec = (struct rte_flow_item_tag){
.data = flow_hw_tx_tag_regc_value(dev),
};
job->items[0] = (struct rte_flow_item){
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
.spec = &job->tag_spec,
};
rte_memcpy(&job->items[1], items, sizeof(*items) * pt->orig_item_nb);
return job->items;
} else {
return items;
}
}
/**
* Enqueue HW steering flow creation.
*
* The flow will be applied to the HW only if the postpone bit is not set or
* the extra push function is called.
* The flow creation status should be checked from dequeue result.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* The queue to create the flow.
* @param[in] attr
* Pointer to the flow operation attributes.
* @param[in] items
* Items with flow spec value.
* @param[in] pattern_template_index
* The item pattern flow follows from the table.
* @param[in] actions
* Action with flow spec value.
* @param[in] action_template_index
* The action pattern flow follows from the table.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* Flow pointer on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow *
flow_hw_async_flow_create(struct rte_eth_dev *dev,
uint32_t queue,
const struct rte_flow_op_attr *attr,
struct rte_flow_template_table *table,
const struct rte_flow_item items[],
uint8_t pattern_template_index,
const struct rte_flow_action actions[],
uint8_t action_template_index,
void *user_data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5dr_rule_attr rule_attr = {
.queue_id = queue,
.user_data = user_data,
.burst = attr->postpone,
};
struct mlx5dr_rule_action rule_acts[MLX5_HW_MAX_ACTS];
struct rte_flow_hw *flow;
struct mlx5_hw_q_job *job;
const struct rte_flow_item *rule_items;
uint32_t flow_idx;
int ret;
if (unlikely((!dev->data->dev_started))) {
rte_errno = EINVAL;
goto error;
}
if (unlikely(!priv->hw_q[queue].job_idx)) {
rte_errno = ENOMEM;
goto error;
}
flow = mlx5_ipool_zmalloc(table->flow, &flow_idx);
if (!flow)
goto error;
/*
* Set the table here in order to know the destination table
* when free the flow afterwards.
*/
flow->table = table;
flow->idx = flow_idx;
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
/*
* Set the job type here in order to know if the flow memory
* should be freed or not when get the result from dequeue.
*/
job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
job->flow = flow;
job->user_data = user_data;
rule_attr.user_data = job;
/*
* Indexed pool returns 1-based indices, but mlx5dr expects 0-based indices for rule
* insertion hints.
*/
MLX5_ASSERT(flow_idx > 0);
rule_attr.rule_idx = flow_idx - 1;
/*
* Construct the flow actions based on the input actions.
* The implicitly appended action is always fixed, like metadata
* copy action from FDB to NIC Rx.
* No need to copy and contrust a new "actions" list based on the
* user's input, in order to save the cost.
*/
if (flow_hw_actions_construct(dev, job,
&table->ats[action_template_index],
pattern_template_index, actions,
rule_acts, queue, error)) {
rte_errno = EINVAL;
goto free;
}
rule_items = flow_hw_get_rule_items(dev, table, items,
pattern_template_index, job);
if (!rule_items)
goto free;
ret = mlx5dr_rule_create(table->matcher,
pattern_template_index, rule_items,
action_template_index, rule_acts,
&rule_attr, (struct mlx5dr_rule *)flow->rule);
if (likely(!ret))
return (struct rte_flow *)flow;
free:
/* Flow created fail, return the descriptor and flow memory. */
mlx5_ipool_free(table->flow, flow_idx);
priv->hw_q[queue].job_idx++;
error:
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to create rte flow");
return NULL;
}
/**
* Enqueue HW steering flow destruction.
*
* The flow will be applied to the HW only if the postpone bit is not set or
* the extra push function is called.
* The flow destruction status should be checked from dequeue result.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* The queue to destroy the flow.
* @param[in] attr
* Pointer to the flow operation attributes.
* @param[in] flow
* Pointer to the flow to be destroyed.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_async_flow_destroy(struct rte_eth_dev *dev,
uint32_t queue,
const struct rte_flow_op_attr *attr,
struct rte_flow *flow,
void *user_data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5dr_rule_attr rule_attr = {
.queue_id = queue,
.user_data = user_data,
.burst = attr->postpone,
};
struct rte_flow_hw *fh = (struct rte_flow_hw *)flow;
struct mlx5_hw_q_job *job;
int ret;
if (unlikely(!priv->hw_q[queue].job_idx)) {
rte_errno = ENOMEM;
goto error;
}
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
job->user_data = user_data;
job->flow = fh;
rule_attr.user_data = job;
ret = mlx5dr_rule_destroy((struct mlx5dr_rule *)fh->rule, &rule_attr);
if (likely(!ret))
return 0;
priv->hw_q[queue].job_idx++;
error:
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to create rte flow");
}
/**
* Release the AGE and counter for given flow.
*
* @param[in] priv
* Pointer to the port private data structure.
* @param[in] queue
* The queue to release the counter.
* @param[in, out] flow
* Pointer to the flow containing the counter.
* @param[out] error
* Pointer to error structure.
*/
static void
flow_hw_age_count_release(struct mlx5_priv *priv, uint32_t queue,
struct rte_flow_hw *flow,
struct rte_flow_error *error)
{
if (mlx5_hws_cnt_is_shared(priv->hws_cpool, flow->cnt_id)) {
if (flow->age_idx && !mlx5_hws_age_is_indirect(flow->age_idx)) {
/* Remove this AGE parameter from indirect counter. */
mlx5_hws_cnt_age_set(priv->hws_cpool, flow->cnt_id, 0);
/* Release the AGE parameter. */
mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
flow->age_idx = 0;
}
return;
}
/* Put the counter first to reduce the race risk in BG thread. */
mlx5_hws_cnt_pool_put(priv->hws_cpool, &queue, &flow->cnt_id);
flow->cnt_id = 0;
if (flow->age_idx) {
if (mlx5_hws_age_is_indirect(flow->age_idx)) {
uint32_t idx = flow->age_idx & MLX5_HWS_AGE_IDX_MASK;
mlx5_hws_age_nb_cnt_decrease(priv, idx);
} else {
/* Release the AGE parameter. */
mlx5_hws_age_action_destroy(priv, flow->age_idx, error);
}
flow->age_idx = 0;
}
}
static inline int
__flow_hw_pull_indir_action_comp(struct rte_eth_dev *dev,
uint32_t queue,
struct rte_flow_op_result res[],
uint16_t n_res)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_ring *r = priv->hw_q[queue].indir_cq;
struct mlx5_hw_q_job *job;
void *user_data = NULL;
uint32_t type, idx;
struct mlx5_aso_mtr *aso_mtr;
struct mlx5_aso_ct_action *aso_ct;
int ret_comp, i;
ret_comp = (int)rte_ring_count(r);
if (ret_comp > n_res)
ret_comp = n_res;
for (i = 0; i < ret_comp; i++) {
rte_ring_dequeue(r, &user_data);
res[i].user_data = user_data;
res[i].status = RTE_FLOW_OP_SUCCESS;
}
if (ret_comp < n_res && priv->hws_mpool)
ret_comp += mlx5_aso_pull_completion(&priv->hws_mpool->sq[queue],
&res[ret_comp], n_res - ret_comp);
if (ret_comp < n_res && priv->hws_ctpool)
ret_comp += mlx5_aso_pull_completion(&priv->ct_mng->aso_sqs[queue],
&res[ret_comp], n_res - ret_comp);
for (i = 0; i < ret_comp; i++) {
job = (struct mlx5_hw_q_job *)res[i].user_data;
/* Restore user data. */
res[i].user_data = job->user_data;
if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
mlx5_ipool_free(priv->hws_mpool->idx_pool, idx);
}
} else if (job->type == MLX5_HW_Q_JOB_TYPE_CREATE) {
type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
if (type == MLX5_INDIRECT_ACTION_TYPE_METER_MARK) {
idx = MLX5_INDIRECT_ACTION_IDX_GET(job->action);
aso_mtr = mlx5_ipool_get(priv->hws_mpool->idx_pool, idx);
aso_mtr->state = ASO_METER_READY;
} else if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
idx = MLX5_ACTION_CTX_CT_GET_IDX
((uint32_t)(uintptr_t)job->action);
aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
aso_ct->state = ASO_CONNTRACK_READY;
}
} else if (job->type == MLX5_HW_Q_JOB_TYPE_QUERY) {
type = MLX5_INDIRECT_ACTION_TYPE_GET(job->action);
if (type == MLX5_INDIRECT_ACTION_TYPE_CT) {
idx = MLX5_ACTION_CTX_CT_GET_IDX
((uint32_t)(uintptr_t)job->action);
aso_ct = mlx5_ipool_get(priv->hws_ctpool->cts, idx);
mlx5_aso_ct_obj_analyze(job->profile,
job->out_data);
aso_ct->state = ASO_CONNTRACK_READY;
}
}
priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
}
return ret_comp;
}
/**
* Pull the enqueued flows.
*
* For flows enqueued from creation/destruction, the status should be
* checked from the dequeue result.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* The queue to pull the result.
* @param[in/out] res
* Array to save the results.
* @param[in] n_res
* Available result with the array.
* @param[out] error
* Pointer to error structure.
*
* @return
* Result number on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_pull(struct rte_eth_dev *dev,
uint32_t queue,
struct rte_flow_op_result res[],
uint16_t n_res,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
struct mlx5_hw_q_job *job;
int ret, i;
/* 1. Pull the flow completion. */
ret = mlx5dr_send_queue_poll(priv->dr_ctx, queue, res, n_res);
if (ret < 0)
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to query flow queue");
for (i = 0; i < ret; i++) {
job = (struct mlx5_hw_q_job *)res[i].user_data;
/* Restore user data. */
res[i].user_data = job->user_data;
if (job->type == MLX5_HW_Q_JOB_TYPE_DESTROY) {
if (job->flow->fate_type == MLX5_FLOW_FATE_JUMP)
flow_hw_jump_release(dev, job->flow->jump);
else if (job->flow->fate_type == MLX5_FLOW_FATE_QUEUE)
mlx5_hrxq_obj_release(dev, job->flow->hrxq);
if (mlx5_hws_cnt_id_valid(job->flow->cnt_id))
flow_hw_age_count_release(priv, queue,
job->flow, error);
if (job->flow->mtr_id) {
mlx5_ipool_free(pool->idx_pool, job->flow->mtr_id);
job->flow->mtr_id = 0;
}
mlx5_ipool_free(job->flow->table->flow, job->flow->idx);
}
priv->hw_q[queue].job[priv->hw_q[queue].job_idx++] = job;
}
/* 2. Pull indirect action comp. */
if (ret < n_res)
ret += __flow_hw_pull_indir_action_comp(dev, queue, &res[ret],
n_res - ret);
return ret;
}
static inline void
__flow_hw_push_action(struct rte_eth_dev *dev,
uint32_t queue)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_ring *iq = priv->hw_q[queue].indir_iq;
struct rte_ring *cq = priv->hw_q[queue].indir_cq;
void *job = NULL;
uint32_t ret, i;
ret = rte_ring_count(iq);
for (i = 0; i < ret; i++) {
rte_ring_dequeue(iq, &job);
rte_ring_enqueue(cq, job);
}
if (priv->hws_ctpool)
mlx5_aso_push_wqe(priv->sh, &priv->ct_mng->aso_sqs[queue]);
if (priv->hws_mpool)
mlx5_aso_push_wqe(priv->sh, &priv->hws_mpool->sq[queue]);
}
/**
* Push the enqueued flows to HW.
*
* Force apply all the enqueued flows to the HW.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* The queue to push the flow.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_push(struct rte_eth_dev *dev,
uint32_t queue,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
int ret;
__flow_hw_push_action(dev, queue);
ret = mlx5dr_send_queue_action(priv->dr_ctx, queue,
MLX5DR_SEND_QUEUE_ACTION_DRAIN);
if (ret) {
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to push flows");
return ret;
}
return 0;
}
/**
* Drain the enqueued flows' completion.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* The queue to pull the flow.
* @param[in] pending_rules
* The pending flow number.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
__flow_hw_pull_comp(struct rte_eth_dev *dev,
uint32_t queue,
uint32_t pending_rules,
struct rte_flow_error *error)
{
struct rte_flow_op_result comp[BURST_THR];
int ret, i, empty_loop = 0;
ret = flow_hw_push(dev, queue, error);
if (ret < 0)
return ret;
while (pending_rules) {
ret = flow_hw_pull(dev, queue, comp, BURST_THR, error);
if (ret < 0)
return -1;
if (!ret) {
rte_delay_us_sleep(20000);
if (++empty_loop > 5) {
DRV_LOG(WARNING, "No available dequeue, quit.");
break;
}
continue;
}
for (i = 0; i < ret; i++) {
if (comp[i].status == RTE_FLOW_OP_ERROR)
DRV_LOG(WARNING, "Flow flush get error CQE.");
}
if ((uint32_t)ret > pending_rules) {
DRV_LOG(WARNING, "Flow flush get extra CQE.");
return rte_flow_error_set(error, ERANGE,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"get extra CQE");
}
pending_rules -= ret;
empty_loop = 0;
}
return 0;
}
/**
* Flush created flows.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
int
flow_hw_q_flow_flush(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hw_q *hw_q;
struct rte_flow_template_table *tbl;
struct rte_flow_hw *flow;
struct rte_flow_op_attr attr = {
.postpone = 0,
};
uint32_t pending_rules = 0;
uint32_t queue;
uint32_t fidx;
/*
* Ensure to push and dequeue all the enqueued flow
* creation/destruction jobs in case user forgot to
* dequeue. Or the enqueued created flows will be
* leaked. The forgotten dequeues would also cause
* flow flush get extra CQEs as expected and pending_rules
* be minus value.
*/
for (queue = 0; queue < priv->nb_queue; queue++) {
hw_q = &priv->hw_q[queue];
if (__flow_hw_pull_comp(dev, queue, hw_q->size - hw_q->job_idx,
error))
return -1;
}
/* Flush flow per-table from MLX5_DEFAULT_FLUSH_QUEUE. */
hw_q = &priv->hw_q[MLX5_DEFAULT_FLUSH_QUEUE];
LIST_FOREACH(tbl, &priv->flow_hw_tbl, next) {
if (!tbl->cfg.external)
continue;
MLX5_IPOOL_FOREACH(tbl->flow, fidx, flow) {
if (flow_hw_async_flow_destroy(dev,
MLX5_DEFAULT_FLUSH_QUEUE,
&attr,
(struct rte_flow *)flow,
NULL,
error))
return -1;
pending_rules++;
/* Drain completion with queue size. */
if (pending_rules >= hw_q->size) {
if (__flow_hw_pull_comp(dev,
MLX5_DEFAULT_FLUSH_QUEUE,
pending_rules, error))
return -1;
pending_rules = 0;
}
}
}
/* Drain left completion. */
if (pending_rules &&
__flow_hw_pull_comp(dev, MLX5_DEFAULT_FLUSH_QUEUE, pending_rules,
error))
return -1;
return 0;
}
/**
* Create flow table.
*
* The input item and action templates will be binded to the table.
* Flow memory will also be allocated. Matcher will be created based
* on the item template. Action will be translated to the dedicated
* DR action if possible.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] table_cfg
* Pointer to the table configuration.
* @param[in] item_templates
* Item template array to be binded to the table.
* @param[in] nb_item_templates
* Number of item template.
* @param[in] action_templates
* Action template array to be binded to the table.
* @param[in] nb_action_templates
* Number of action template.
* @param[out] error
* Pointer to error structure.
*
* @return
* Table on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow_template_table *
flow_hw_table_create(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *table_cfg,
struct rte_flow_pattern_template *item_templates[],
uint8_t nb_item_templates,
struct rte_flow_actions_template *action_templates[],
uint8_t nb_action_templates,
struct rte_flow_error *error)
{
struct rte_flow_error sub_error = {
.type = RTE_FLOW_ERROR_TYPE_NONE,
.cause = NULL,
.message = NULL,
};
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5dr_matcher_attr matcher_attr = {0};
struct rte_flow_template_table *tbl = NULL;
struct mlx5_flow_group *grp;
struct mlx5dr_match_template *mt[MLX5_HW_TBL_MAX_ITEM_TEMPLATE];
struct mlx5dr_action_template *at[MLX5_HW_TBL_MAX_ACTION_TEMPLATE];
const struct rte_flow_template_table_attr *attr = &table_cfg->attr;
struct rte_flow_attr flow_attr = attr->flow_attr;
struct mlx5_flow_cb_ctx ctx = {
.dev = dev,
.error = &sub_error,
.data = &flow_attr,
};
struct mlx5_indexed_pool_config cfg = {
.size = sizeof(struct rte_flow_hw) + mlx5dr_rule_get_handle_size(),
.trunk_size = 1 << 12,
.per_core_cache = 1 << 13,
.need_lock = 1,
.release_mem_en = !!priv->sh->config.reclaim_mode,
.malloc = mlx5_malloc,
.free = mlx5_free,
.type = "mlx5_hw_table_flow",
};
struct mlx5_list_entry *ge;
uint32_t i, max_tpl = MLX5_HW_TBL_MAX_ITEM_TEMPLATE;
uint32_t nb_flows = rte_align32pow2(attr->nb_flows);
bool port_started = !!dev->data->dev_started;
int err;
/* HWS layer accepts only 1 item template with root table. */
if (!attr->flow_attr.group)
max_tpl = 1;
cfg.max_idx = nb_flows;
/* For table has very limited flows, disable cache. */
if (nb_flows < cfg.trunk_size) {
cfg.per_core_cache = 0;
cfg.trunk_size = nb_flows;
} else if (nb_flows <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
}
/* Check if we requires too many templates. */
if (nb_item_templates > max_tpl ||
nb_action_templates > MLX5_HW_TBL_MAX_ACTION_TEMPLATE) {
rte_errno = EINVAL;
goto error;
}
/* Allocate the table memory. */
tbl = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl), 0, rte_socket_id());
if (!tbl)
goto error;
tbl->cfg = *table_cfg;
/* Allocate flow indexed pool. */
tbl->flow = mlx5_ipool_create(&cfg);
if (!tbl->flow)
goto error;
/* Register the flow group. */
ge = mlx5_hlist_register(priv->sh->groups, attr->flow_attr.group, &ctx);
if (!ge)
goto error;
grp = container_of(ge, struct mlx5_flow_group, entry);
tbl->grp = grp;
/* Prepare matcher information. */
matcher_attr.priority = attr->flow_attr.priority;
matcher_attr.optimize_using_rule_idx = true;
matcher_attr.mode = MLX5DR_MATCHER_RESOURCE_MODE_RULE;
matcher_attr.rule.num_log = rte_log2_u32(nb_flows);
/* Build the item template. */
for (i = 0; i < nb_item_templates; i++) {
uint32_t ret;
if ((flow_attr.ingress && !item_templates[i]->attr.ingress) ||
(flow_attr.egress && !item_templates[i]->attr.egress) ||
(flow_attr.transfer && !item_templates[i]->attr.transfer)) {
DRV_LOG(ERR, "pattern template and template table attribute mismatch");
rte_errno = EINVAL;
goto it_error;
}
ret = __atomic_add_fetch(&item_templates[i]->refcnt, 1,
__ATOMIC_RELAXED);
if (ret <= 1) {
rte_errno = EINVAL;
goto it_error;
}
mt[i] = item_templates[i]->mt;
tbl->its[i] = item_templates[i];
}
tbl->nb_item_templates = nb_item_templates;
/* Build the action template. */
for (i = 0; i < nb_action_templates; i++) {
uint32_t ret;
ret = __atomic_add_fetch(&action_templates[i]->refcnt, 1,
__ATOMIC_RELAXED);
if (ret <= 1) {
rte_errno = EINVAL;
goto at_error;
}
at[i] = action_templates[i]->tmpl;
tbl->ats[i].action_template = action_templates[i];
LIST_INIT(&tbl->ats[i].acts.act_list);
if (!port_started)
continue;
err = __flow_hw_actions_translate(dev, &tbl->cfg,
&tbl->ats[i].acts,
action_templates[i], &sub_error);
if (err) {
i++;
goto at_error;
}
}
tbl->nb_action_templates = nb_action_templates;
tbl->matcher = mlx5dr_matcher_create
(tbl->grp->tbl, mt, nb_item_templates, at, nb_action_templates, &matcher_attr);
if (!tbl->matcher)
goto at_error;
tbl->type = attr->flow_attr.transfer ? MLX5DR_TABLE_TYPE_FDB :
(attr->flow_attr.egress ? MLX5DR_TABLE_TYPE_NIC_TX :
MLX5DR_TABLE_TYPE_NIC_RX);
if (port_started)
LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
else
LIST_INSERT_HEAD(&priv->flow_hw_tbl_ongo, tbl, next);
return tbl;
at_error:
while (i--) {
__flow_hw_action_template_destroy(dev, &tbl->ats[i].acts);
__atomic_sub_fetch(&action_templates[i]->refcnt,
1, __ATOMIC_RELAXED);
}
i = nb_item_templates;
it_error:
while (i--)
__atomic_sub_fetch(&item_templates[i]->refcnt,
1, __ATOMIC_RELAXED);
error:
err = rte_errno;
if (tbl) {
if (tbl->grp)
mlx5_hlist_unregister(priv->sh->groups,
&tbl->grp->entry);
if (tbl->flow)
mlx5_ipool_destroy(tbl->flow);
mlx5_free(tbl);
}
if (error != NULL) {
if (sub_error.type == RTE_FLOW_ERROR_TYPE_NONE)
rte_flow_error_set(error, err, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Failed to create template table");
else
rte_memcpy(error, &sub_error, sizeof(sub_error));
}
return NULL;
}
/**
* Update flow template table.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
int
flow_hw_table_update(struct rte_eth_dev *dev,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_template_table *tbl;
while ((tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo)) != NULL) {
if (flow_hw_actions_translate(dev, tbl, error))
return -1;
LIST_REMOVE(tbl, next);
LIST_INSERT_HEAD(&priv->flow_hw_tbl, tbl, next);
}
return 0;
}
/**
* Translates group index specified by the user in @p attr to internal
* group index.
*
* Translation is done by incrementing group index, so group n becomes n + 1.
*
* @param[in] dev
* Pointer to Ethernet device.
* @param[in] cfg
* Pointer to the template table configuration.
* @param[in] group
* Currently used group index (table group or jump destination).
* @param[out] table_group
* Pointer to output group index.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success. Otherwise, returns negative error code, rte_errno is set
* and error structure is filled.
*/
static int
flow_hw_translate_group(struct rte_eth_dev *dev,
const struct mlx5_flow_template_table_cfg *cfg,
uint32_t group,
uint32_t *table_group,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_sh_config *config = &priv->sh->config;
const struct rte_flow_attr *flow_attr = &cfg->attr.flow_attr;
if (config->dv_esw_en &&
priv->fdb_def_rule &&
cfg->external &&
flow_attr->transfer) {
if (group > MLX5_HW_MAX_TRANSFER_GROUP)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
NULL,
"group index not supported");
*table_group = group + 1;
} else if (config->dv_esw_en &&
!(config->repr_matching && config->dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) &&
cfg->external &&
flow_attr->egress) {
/*
* On E-Switch setups, egress group translation is not done if and only if
* representor matching is disabled and legacy metadata mode is selected.
* In all other cases, egree group 0 is reserved for representor tagging flows
* and metadata copy flows.
*/
if (group > MLX5_HW_MAX_EGRESS_GROUP)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR_GROUP,
NULL,
"group index not supported");
*table_group = group + 1;
} else {
*table_group = group;
}
return 0;
}
/**
* Create flow table.
*
* This function is a wrapper over @ref flow_hw_table_create(), which translates parameters
* provided by user to proper internal values.
*
* @param[in] dev
* Pointer to Ethernet device.
* @param[in] attr
* Pointer to the table attributes.
* @param[in] item_templates
* Item template array to be binded to the table.
* @param[in] nb_item_templates
* Number of item templates.
* @param[in] action_templates
* Action template array to be binded to the table.
* @param[in] nb_action_templates
* Number of action templates.
* @param[out] error
* Pointer to error structure.
*
* @return
* Table on success, Otherwise, returns negative error code, rte_errno is set
* and error structure is filled.
*/
static struct rte_flow_template_table *
flow_hw_template_table_create(struct rte_eth_dev *dev,
const struct rte_flow_template_table_attr *attr,
struct rte_flow_pattern_template *item_templates[],
uint8_t nb_item_templates,
struct rte_flow_actions_template *action_templates[],
uint8_t nb_action_templates,
struct rte_flow_error *error)
{
struct mlx5_flow_template_table_cfg cfg = {
.attr = *attr,
.external = true,
};
uint32_t group = attr->flow_attr.group;
if (flow_hw_translate_group(dev, &cfg, group, &cfg.attr.flow_attr.group, error))
return NULL;
return flow_hw_table_create(dev, &cfg, item_templates, nb_item_templates,
action_templates, nb_action_templates, error);
}
/**
* Destroy flow table.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] table
* Pointer to the table to be destroyed.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_table_destroy(struct rte_eth_dev *dev,
struct rte_flow_template_table *table,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
int i;
uint32_t fidx = 1;
/* Build ipool allocated object bitmap. */
mlx5_ipool_flush_cache(table->flow);
/* Check if ipool has allocated objects. */
if (table->refcnt || mlx5_ipool_get_next(table->flow, &fidx)) {
DRV_LOG(WARNING, "Table %p is still in using.", (void *)table);
return rte_flow_error_set(error, EBUSY,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"table in using");
}
LIST_REMOVE(table, next);
for (i = 0; i < table->nb_item_templates; i++)
__atomic_sub_fetch(&table->its[i]->refcnt,
1, __ATOMIC_RELAXED);
for (i = 0; i < table->nb_action_templates; i++) {
__flow_hw_action_template_destroy(dev, &table->ats[i].acts);
__atomic_sub_fetch(&table->ats[i].action_template->refcnt,
1, __ATOMIC_RELAXED);
}
mlx5dr_matcher_destroy(table->matcher);
mlx5_hlist_unregister(priv->sh->groups, &table->grp->entry);
mlx5_ipool_destroy(table->flow);
mlx5_free(table);
return 0;
}
static bool
flow_hw_modify_field_is_used(const struct rte_flow_action_modify_field *action,
enum rte_flow_field_id field)
{
return action->src.field == field || action->dst.field == field;
}
static int
flow_hw_validate_action_modify_field(const struct rte_flow_action *action,
const struct rte_flow_action *mask,
struct rte_flow_error *error)
{
const struct rte_flow_action_modify_field *action_conf =
action->conf;
const struct rte_flow_action_modify_field *mask_conf =
mask->conf;
if (action_conf->operation != mask_conf->operation)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"modify_field operation mask and template are not equal");
if (action_conf->dst.field != mask_conf->dst.field)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"destination field mask and template are not equal");
if (action_conf->dst.field == RTE_FLOW_FIELD_POINTER ||
action_conf->dst.field == RTE_FLOW_FIELD_VALUE)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"immediate value and pointer cannot be used as destination");
if (mask_conf->dst.level != UINT32_MAX)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"destination encapsulation level must be fully masked");
if (mask_conf->dst.offset != UINT32_MAX)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"destination offset level must be fully masked");
if (action_conf->src.field != mask_conf->src.field)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"destination field mask and template are not equal");
if (action_conf->src.field != RTE_FLOW_FIELD_POINTER &&
action_conf->src.field != RTE_FLOW_FIELD_VALUE) {
if (mask_conf->src.level != UINT32_MAX)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"source encapsulation level must be fully masked");
if (mask_conf->src.offset != UINT32_MAX)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"source offset level must be fully masked");
}
if (mask_conf->width != UINT32_MAX)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"modify_field width field must be fully masked");
if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_START))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"modifying arbitrary place in a packet is not supported");
if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_VLAN_TYPE))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"modifying vlan_type is not supported");
if (flow_hw_modify_field_is_used(action_conf, RTE_FLOW_FIELD_GENEVE_VNI))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"modifying Geneve VNI is not supported");
return 0;
}
static int
flow_hw_validate_action_represented_port(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
const struct rte_flow_action *mask,
struct rte_flow_error *error)
{
const struct rte_flow_action_ethdev *action_conf = action->conf;
const struct rte_flow_action_ethdev *mask_conf = mask->conf;
struct mlx5_priv *priv = dev->data->dev_private;
if (!priv->sh->config.dv_esw_en)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"cannot use represented_port actions"
" without an E-Switch");
if (mask_conf && mask_conf->port_id) {
struct mlx5_priv *port_priv;
struct mlx5_priv *dev_priv;
if (!action_conf)
return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ACTION,
action, "port index was not provided");
port_priv = mlx5_port_to_eswitch_info(action_conf->port_id, false);
if (!port_priv)
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_ACTION,
action,
"failed to obtain E-Switch"
" info for port");
dev_priv = mlx5_dev_to_eswitch_info(dev);
if (!dev_priv)
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_ACTION,
action,
"failed to obtain E-Switch"
" info for transfer proxy");
if (port_priv->domain_id != dev_priv->domain_id)
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_ACTION,
action,
"cannot forward to port from"
" a different E-Switch");
}
return 0;
}
/**
* Validate AGE action.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
* @param[in] action
* Pointer to the indirect action.
* @param[in] action_flags
* Holds the actions detected until now.
* @param[in] fixed_cnt
* Indicator if this list has a fixed COUNT action.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_validate_action_age(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
uint64_t action_flags, bool fixed_cnt,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
if (!priv->sh->cdev->config.devx)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "AGE action not supported");
if (age_info->ages_ipool == NULL)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"aging pool not initialized");
if ((action_flags & MLX5_FLOW_ACTION_AGE) ||
(action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"duplicate AGE actions set");
if (fixed_cnt)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"AGE and fixed COUNT combination is not supported");
return 0;
}
/**
* Validate count action.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
* @param[in] action
* Pointer to the indirect action.
* @param[in] mask
* Pointer to the indirect action mask.
* @param[in] action_flags
* Holds the actions detected until now.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_validate_action_count(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
const struct rte_flow_action *mask,
uint64_t action_flags,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_action_count *count = mask->conf;
if (!priv->sh->cdev->config.devx)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"count action not supported");
if (!priv->hws_cpool)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"counters pool not initialized");
if ((action_flags & MLX5_FLOW_ACTION_COUNT) ||
(action_flags & MLX5_FLOW_ACTION_INDIRECT_COUNT))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"duplicate count actions set");
if (count && count->id && (action_flags & MLX5_FLOW_ACTION_AGE))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, mask,
"AGE and COUNT action shared by mask combination is not supported");
return 0;
}
/**
* Validate meter_mark action.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
* @param[in] action
* Pointer to the indirect action.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_validate_action_meter_mark(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
RTE_SET_USED(action);
if (!priv->sh->cdev->config.devx)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"meter_mark action not supported");
if (!priv->hws_mpool)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"meter_mark pool not initialized");
return 0;
}
/**
* Validate indirect action.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
* @param[in] action
* Pointer to the indirect action.
* @param[in] mask
* Pointer to the indirect action mask.
* @param[in, out] action_flags
* Holds the actions detected until now.
* @param[in, out] fixed_cnt
* Pointer to indicator if this list has a fixed COUNT action.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_validate_action_indirect(struct rte_eth_dev *dev,
const struct rte_flow_action *action,
const struct rte_flow_action *mask,
uint64_t *action_flags, bool *fixed_cnt,
struct rte_flow_error *error)
{
uint32_t type;
int ret;
if (!mask)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Unable to determine indirect action type without a mask specified");
type = mask->type;
switch (type) {
case RTE_FLOW_ACTION_TYPE_METER_MARK:
ret = flow_hw_validate_action_meter_mark(dev, mask, error);
if (ret < 0)
return ret;
*action_flags |= MLX5_FLOW_ACTION_METER;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
/* TODO: Validation logic (same as flow_hw_actions_validate) */
*action_flags |= MLX5_FLOW_ACTION_RSS;
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
/* TODO: Validation logic (same as flow_hw_actions_validate) */
*action_flags |= MLX5_FLOW_ACTION_CT;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
if (action->conf && mask->conf) {
if ((*action_flags & MLX5_FLOW_ACTION_AGE) ||
(*action_flags & MLX5_FLOW_ACTION_INDIRECT_AGE))
/*
* AGE cannot use indirect counter which is
* shared with enother flow rules.
*/
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"AGE and fixed COUNT combination is not supported");
*fixed_cnt = true;
}
ret = flow_hw_validate_action_count(dev, action, mask,
*action_flags, error);
if (ret < 0)
return ret;
*action_flags |= MLX5_FLOW_ACTION_INDIRECT_COUNT;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
ret = flow_hw_validate_action_age(dev, action, *action_flags,
*fixed_cnt, error);
if (ret < 0)
return ret;
*action_flags |= MLX5_FLOW_ACTION_INDIRECT_AGE;
break;
default:
DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, mask,
"Unsupported indirect action type");
}
return 0;
}
/**
* Validate raw_encap action.
*
* @param[in] dev
* Pointer to rte_eth_dev structure.
* @param[in] action
* Pointer to the indirect action.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_validate_action_raw_encap(struct rte_eth_dev *dev __rte_unused,
const struct rte_flow_action *action,
struct rte_flow_error *error)
{
const struct rte_flow_action_raw_encap *raw_encap_data = action->conf;
if (!raw_encap_data || !raw_encap_data->size || !raw_encap_data->data)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"invalid raw_encap_data");
return 0;
}
static inline uint16_t
flow_hw_template_expand_modify_field(const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
const struct rte_flow_action *mf_action,
const struct rte_flow_action *mf_mask,
struct rte_flow_action *new_actions,
struct rte_flow_action *new_masks,
uint64_t flags, uint32_t act_num)
{
uint32_t i, tail;
MLX5_ASSERT(actions && masks);
MLX5_ASSERT(new_actions && new_masks);
MLX5_ASSERT(mf_action && mf_mask);
if (flags & MLX5_FLOW_ACTION_MODIFY_FIELD) {
/*
* Application action template already has Modify Field.
* It's location will be used in DR.
* Expanded MF action can be added before the END.
*/
i = act_num - 1;
goto insert;
}
/**
* Locate the first action positioned BEFORE the new MF.
*
* Search for a place to insert modify header
* from the END action backwards:
* 1. END is always present in actions array
* 2. END location is always at action[act_num - 1]
* 3. END always positioned AFTER modify field location
*
* Relative actions order is the same for RX, TX and FDB.
*
* Current actions order (draft-3)
* @see action_order_arr[]
*/
for (i = act_num - 2; (int)i >= 0; i--) {
enum rte_flow_action_type type = actions[i].type;
if (type == RTE_FLOW_ACTION_TYPE_INDIRECT)
type = masks[i].type;
switch (type) {
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
case RTE_FLOW_ACTION_TYPE_DROP:
case RTE_FLOW_ACTION_TYPE_JUMP:
case RTE_FLOW_ACTION_TYPE_QUEUE:
case RTE_FLOW_ACTION_TYPE_RSS:
case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
case RTE_FLOW_ACTION_TYPE_VOID:
case RTE_FLOW_ACTION_TYPE_END:
break;
default:
i++; /* new MF inserted AFTER actions[i] */
goto insert;
break;
}
}
i = 0;
insert:
tail = act_num - i; /* num action to move */
memcpy(new_actions, actions, sizeof(actions[0]) * i);
new_actions[i] = *mf_action;
memcpy(new_actions + i + 1, actions + i, sizeof(actions[0]) * tail);
memcpy(new_masks, masks, sizeof(masks[0]) * i);
new_masks[i] = *mf_mask;
memcpy(new_masks + i + 1, masks + i, sizeof(masks[0]) * tail);
return i;
}
static int
flow_hw_validate_action_push_vlan(struct rte_eth_dev *dev,
const
struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action *action,
const struct rte_flow_action *mask,
struct rte_flow_error *error)
{
#define X_FIELD(ptr, t, f) (((ptr)->conf) && ((t *)((ptr)->conf))->f)
const bool masked_push =
X_FIELD(mask + MLX5_HW_VLAN_PUSH_TYPE_IDX,
const struct rte_flow_action_of_push_vlan, ethertype);
bool masked_param;
/*
* Mandatory actions order:
* OF_PUSH_VLAN / OF_SET_VLAN_VID [ / OF_SET_VLAN_PCP ]
*/
RTE_SET_USED(dev);
RTE_SET_USED(attr);
/* Check that mark matches OF_PUSH_VLAN */
if (mask[MLX5_HW_VLAN_PUSH_TYPE_IDX].type !=
RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
action, "OF_PUSH_VLAN: mask does not match");
/* Check that the second template and mask items are SET_VLAN_VID */
if (action[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID ||
mask[MLX5_HW_VLAN_PUSH_VID_IDX].type !=
RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
action, "OF_PUSH_VLAN: invalid actions order");
masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_VID_IDX,
const struct rte_flow_action_of_set_vlan_vid,
vlan_vid);
/*
* PMD requires OF_SET_VLAN_VID mask to must match OF_PUSH_VLAN
*/
if (masked_push ^ masked_param)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"OF_SET_VLAN_VID: mask does not match OF_PUSH_VLAN");
if (is_of_vlan_pcp_present(action)) {
if (mask[MLX5_HW_VLAN_PUSH_PCP_IDX].type !=
RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_PCP)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
action, "OF_SET_VLAN_PCP: missing mask configuration");
masked_param = X_FIELD(mask + MLX5_HW_VLAN_PUSH_PCP_IDX,
const struct
rte_flow_action_of_set_vlan_pcp,
vlan_pcp);
/*
* PMD requires OF_SET_VLAN_PCP mask to must match OF_PUSH_VLAN
*/
if (masked_push ^ masked_param)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, action,
"OF_SET_VLAN_PCP: mask does not match OF_PUSH_VLAN");
}
return 0;
#undef X_FIELD
}
static int
mlx5_flow_hw_actions_validate(struct rte_eth_dev *dev,
const struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
uint64_t *act_flags,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_action_count *count_mask = NULL;
bool fixed_cnt = false;
uint64_t action_flags = 0;
uint16_t i;
bool actions_end = false;
int ret;
/* FDB actions are only valid to proxy port. */
if (attr->transfer && (!priv->sh->config.dv_esw_en || !priv->master))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"transfer actions are only valid to proxy port");
for (i = 0; !actions_end; ++i) {
const struct rte_flow_action *action = &actions[i];
const struct rte_flow_action *mask = &masks[i];
MLX5_ASSERT(i < MLX5_HW_MAX_ACTS);
if (action->type != RTE_FLOW_ACTION_TYPE_INDIRECT &&
action->type != mask->type)
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
action,
"mask type does not match action type");
switch (action->type) {
case RTE_FLOW_ACTION_TYPE_VOID:
break;
case RTE_FLOW_ACTION_TYPE_INDIRECT:
ret = flow_hw_validate_action_indirect(dev, action,
mask,
&action_flags,
&fixed_cnt,
error);
if (ret < 0)
return ret;
break;
case RTE_FLOW_ACTION_TYPE_MARK:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_MARK;
break;
case RTE_FLOW_ACTION_TYPE_DROP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_DROP;
break;
case RTE_FLOW_ACTION_TYPE_JUMP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_JUMP;
break;
case RTE_FLOW_ACTION_TYPE_QUEUE:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_QUEUE;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_RSS;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_ENCAP;
break;
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_ENCAP;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_DECAP;
break;
case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_DECAP;
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
ret = flow_hw_validate_action_raw_encap(dev, action, error);
if (ret < 0)
return ret;
action_flags |= MLX5_FLOW_ACTION_ENCAP;
break;
case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_DECAP;
break;
case RTE_FLOW_ACTION_TYPE_METER:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_METER;
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
ret = flow_hw_validate_action_meter_mark(dev, action,
error);
if (ret < 0)
return ret;
action_flags |= MLX5_FLOW_ACTION_METER;
break;
case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
ret = flow_hw_validate_action_modify_field(action,
mask,
error);
if (ret < 0)
return ret;
action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
break;
case RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT:
ret = flow_hw_validate_action_represented_port
(dev, action, mask, error);
if (ret < 0)
return ret;
action_flags |= MLX5_FLOW_ACTION_PORT_ID;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
if (count_mask && count_mask->id)
fixed_cnt = true;
ret = flow_hw_validate_action_age(dev, action,
action_flags,
fixed_cnt, error);
if (ret < 0)
return ret;
action_flags |= MLX5_FLOW_ACTION_AGE;
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
ret = flow_hw_validate_action_count(dev, action, mask,
action_flags,
error);
if (ret < 0)
return ret;
count_mask = mask->conf;
action_flags |= MLX5_FLOW_ACTION_COUNT;
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
/* TODO: Validation logic */
action_flags |= MLX5_FLOW_ACTION_CT;
break;
case RTE_FLOW_ACTION_TYPE_OF_POP_VLAN:
action_flags |= MLX5_FLOW_ACTION_OF_POP_VLAN;
break;
case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
action_flags |= MLX5_FLOW_ACTION_OF_SET_VLAN_VID;
break;
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
ret = flow_hw_validate_action_push_vlan
(dev, attr, action, mask, error);
if (ret != 0)
return ret;
i += is_of_vlan_pcp_present(action) ?
MLX5_HW_VLAN_PUSH_PCP_IDX :
MLX5_HW_VLAN_PUSH_VID_IDX;
action_flags |= MLX5_FLOW_ACTION_OF_PUSH_VLAN;
break;
case RTE_FLOW_ACTION_TYPE_END:
actions_end = true;
break;
default:
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
action,
"action not supported in template API");
}
}
if (act_flags != NULL)
*act_flags = action_flags;
return 0;
}
static int
flow_hw_actions_validate(struct rte_eth_dev *dev,
const struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
struct rte_flow_error *error)
{
return mlx5_flow_hw_actions_validate(dev, attr, actions, masks, NULL,
error);
}
static enum mlx5dr_action_type mlx5_hw_dr_action_types[] = {
[RTE_FLOW_ACTION_TYPE_MARK] = MLX5DR_ACTION_TYP_TAG,
[RTE_FLOW_ACTION_TYPE_DROP] = MLX5DR_ACTION_TYP_DROP,
[RTE_FLOW_ACTION_TYPE_JUMP] = MLX5DR_ACTION_TYP_FT,
[RTE_FLOW_ACTION_TYPE_QUEUE] = MLX5DR_ACTION_TYP_TIR,
[RTE_FLOW_ACTION_TYPE_RSS] = MLX5DR_ACTION_TYP_TIR,
[RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP] = MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
[RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP] = MLX5DR_ACTION_TYP_L2_TO_TNL_L2,
[RTE_FLOW_ACTION_TYPE_VXLAN_DECAP] = MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
[RTE_FLOW_ACTION_TYPE_NVGRE_DECAP] = MLX5DR_ACTION_TYP_TNL_L2_TO_L2,
[RTE_FLOW_ACTION_TYPE_MODIFY_FIELD] = MLX5DR_ACTION_TYP_MODIFY_HDR,
[RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT] = MLX5DR_ACTION_TYP_VPORT,
[RTE_FLOW_ACTION_TYPE_CONNTRACK] = MLX5DR_ACTION_TYP_ASO_CT,
[RTE_FLOW_ACTION_TYPE_OF_POP_VLAN] = MLX5DR_ACTION_TYP_POP_VLAN,
[RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN] = MLX5DR_ACTION_TYP_PUSH_VLAN,
};
static int
flow_hw_dr_actions_template_handle_shared(const struct rte_flow_action *mask,
unsigned int action_src,
enum mlx5dr_action_type *action_types,
uint16_t *curr_off, uint16_t *cnt_off,
struct rte_flow_actions_template *at)
{
uint32_t type;
if (!mask) {
DRV_LOG(WARNING, "Unable to determine indirect action type "
"without a mask specified");
return -EINVAL;
}
type = mask->type;
switch (type) {
case RTE_FLOW_ACTION_TYPE_RSS:
at->actions_off[action_src] = *curr_off;
action_types[*curr_off] = MLX5DR_ACTION_TYP_TIR;
*curr_off = *curr_off + 1;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
case RTE_FLOW_ACTION_TYPE_COUNT:
/*
* Both AGE and COUNT action need counter, the first one fills
* the action_types array, and the second only saves the offset.
*/
if (*cnt_off == UINT16_MAX) {
*cnt_off = *curr_off;
action_types[*cnt_off] = MLX5DR_ACTION_TYP_CTR;
*curr_off = *curr_off + 1;
}
at->actions_off[action_src] = *cnt_off;
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
at->actions_off[action_src] = *curr_off;
action_types[*curr_off] = MLX5DR_ACTION_TYP_ASO_CT;
*curr_off = *curr_off + 1;
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
at->actions_off[action_src] = *curr_off;
action_types[*curr_off] = MLX5DR_ACTION_TYP_ASO_METER;
*curr_off = *curr_off + 1;
break;
default:
DRV_LOG(WARNING, "Unsupported shared action type: %d", type);
return -EINVAL;
}
return 0;
}
/**
* Create DR action template based on a provided sequence of flow actions.
*
* @param[in] at
* Pointer to flow actions template to be updated.
*
* @return
* DR action template pointer on success and action offsets in @p at are updated.
* NULL otherwise.
*/
static struct mlx5dr_action_template *
flow_hw_dr_actions_template_create(struct rte_flow_actions_template *at)
{
struct mlx5dr_action_template *dr_template;
enum mlx5dr_action_type action_types[MLX5_HW_MAX_ACTS] = { MLX5DR_ACTION_TYP_LAST };
unsigned int i;
uint16_t curr_off;
enum mlx5dr_action_type reformat_act_type = MLX5DR_ACTION_TYP_TNL_L2_TO_L2;
uint16_t reformat_off = UINT16_MAX;
uint16_t mhdr_off = UINT16_MAX;
uint16_t cnt_off = UINT16_MAX;
int ret;
for (i = 0, curr_off = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
const struct rte_flow_action_raw_encap *raw_encap_data;
size_t data_size;
enum mlx5dr_action_type type;
if (curr_off >= MLX5_HW_MAX_ACTS)
goto err_actions_num;
switch (at->actions[i].type) {
case RTE_FLOW_ACTION_TYPE_VOID:
break;
case RTE_FLOW_ACTION_TYPE_INDIRECT:
ret = flow_hw_dr_actions_template_handle_shared
(&at->masks[i],
i,
action_types,
&curr_off,
&cnt_off, at);
if (ret)
return NULL;
break;
case RTE_FLOW_ACTION_TYPE_VXLAN_ENCAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_ENCAP:
case RTE_FLOW_ACTION_TYPE_VXLAN_DECAP:
case RTE_FLOW_ACTION_TYPE_NVGRE_DECAP:
MLX5_ASSERT(reformat_off == UINT16_MAX);
reformat_off = curr_off++;
reformat_act_type = mlx5_hw_dr_action_types[at->actions[i].type];
break;
case RTE_FLOW_ACTION_TYPE_RAW_ENCAP:
raw_encap_data = at->actions[i].conf;
data_size = raw_encap_data->size;
if (reformat_off != UINT16_MAX) {
reformat_act_type = data_size < MLX5_ENCAPSULATION_DECISION_SIZE ?
MLX5DR_ACTION_TYP_TNL_L3_TO_L2 :
MLX5DR_ACTION_TYP_L2_TO_TNL_L3;
} else {
reformat_off = curr_off++;
reformat_act_type = MLX5DR_ACTION_TYP_L2_TO_TNL_L2;
}
break;
case RTE_FLOW_ACTION_TYPE_RAW_DECAP:
reformat_off = curr_off++;
reformat_act_type = MLX5DR_ACTION_TYP_TNL_L2_TO_L2;
break;
case RTE_FLOW_ACTION_TYPE_MODIFY_FIELD:
if (mhdr_off == UINT16_MAX) {
mhdr_off = curr_off++;
type = mlx5_hw_dr_action_types[at->actions[i].type];
action_types[mhdr_off] = type;
}
break;
case RTE_FLOW_ACTION_TYPE_METER:
at->actions_off[i] = curr_off;
action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
if (curr_off >= MLX5_HW_MAX_ACTS)
goto err_actions_num;
action_types[curr_off++] = MLX5DR_ACTION_TYP_FT;
break;
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
type = mlx5_hw_dr_action_types[at->actions[i].type];
at->actions_off[i] = curr_off;
action_types[curr_off++] = type;
i += is_of_vlan_pcp_present(at->actions + i) ?
MLX5_HW_VLAN_PUSH_PCP_IDX :
MLX5_HW_VLAN_PUSH_VID_IDX;
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
at->actions_off[i] = curr_off;
action_types[curr_off++] = MLX5DR_ACTION_TYP_ASO_METER;
if (curr_off >= MLX5_HW_MAX_ACTS)
goto err_actions_num;
break;
case RTE_FLOW_ACTION_TYPE_AGE:
case RTE_FLOW_ACTION_TYPE_COUNT:
/*
* Both AGE and COUNT action need counter, the first
* one fills the action_types array, and the second only
* saves the offset.
*/
if (cnt_off == UINT16_MAX) {
cnt_off = curr_off++;
action_types[cnt_off] = MLX5DR_ACTION_TYP_CTR;
}
at->actions_off[i] = cnt_off;
break;
default:
type = mlx5_hw_dr_action_types[at->actions[i].type];
at->actions_off[i] = curr_off;
action_types[curr_off++] = type;
break;
}
}
if (curr_off >= MLX5_HW_MAX_ACTS)
goto err_actions_num;
if (mhdr_off != UINT16_MAX)
at->mhdr_off = mhdr_off;
if (reformat_off != UINT16_MAX) {
at->reformat_off = reformat_off;
action_types[reformat_off] = reformat_act_type;
}
dr_template = mlx5dr_action_template_create(action_types);
if (dr_template)
at->dr_actions_num = curr_off;
else
DRV_LOG(ERR, "Failed to create DR action template: %d", rte_errno);
return dr_template;
err_actions_num:
DRV_LOG(ERR, "Number of HW actions (%u) exceeded maximum (%u) allowed in template",
curr_off, MLX5_HW_MAX_ACTS);
return NULL;
}
static void
flow_hw_set_vlan_vid(struct rte_eth_dev *dev,
struct rte_flow_action *ra,
struct rte_flow_action *rm,
struct rte_flow_action_modify_field *spec,
struct rte_flow_action_modify_field *mask,
int set_vlan_vid_ix)
{
struct rte_flow_error error;
const bool masked = rm[set_vlan_vid_ix].conf &&
(((const struct rte_flow_action_of_set_vlan_vid *)
rm[set_vlan_vid_ix].conf)->vlan_vid != 0);
const struct rte_flow_action_of_set_vlan_vid *conf =
ra[set_vlan_vid_ix].conf;
rte_be16_t vid = masked ? conf->vlan_vid : 0;
int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
NULL, &error);
*spec = (typeof(*spec)) {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = RTE_FLOW_FIELD_VLAN_ID,
.level = 0, .offset = 0,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
.level = vid,
.offset = 0,
},
.width = width,
};
*mask = (typeof(*mask)) {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = RTE_FLOW_FIELD_VLAN_ID,
.level = 0xffffffff, .offset = 0xffffffff,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
.level = masked ? (1U << width) - 1 : 0,
.offset = 0,
},
.width = 0xffffffff,
};
ra[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
ra[set_vlan_vid_ix].conf = spec;
rm[set_vlan_vid_ix].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
rm[set_vlan_vid_ix].conf = mask;
}
static __rte_always_inline int
flow_hw_set_vlan_vid_construct(struct rte_eth_dev *dev,
struct mlx5_hw_q_job *job,
struct mlx5_action_construct_data *act_data,
const struct mlx5_hw_actions *hw_acts,
const struct rte_flow_action *action)
{
struct rte_flow_error error;
rte_be16_t vid = ((const struct rte_flow_action_of_set_vlan_vid *)
action->conf)->vlan_vid;
int width = mlx5_flow_item_field_width(dev, RTE_FLOW_FIELD_VLAN_ID, 0,
NULL, &error);
struct rte_flow_action_modify_field conf = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = RTE_FLOW_FIELD_VLAN_ID,
.level = 0, .offset = 0,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
.level = vid,
.offset = 0,
},
.width = width,
};
struct rte_flow_action modify_action = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &conf
};
return flow_hw_modify_field_construct(job, act_data, hw_acts,
&modify_action);
}
/**
* Create flow action template.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] attr
* Pointer to the action template attributes.
* @param[in] actions
* Associated actions (list terminated by the END action).
* @param[in] masks
* List of actions that marks which of the action's member is constant.
* @param[out] error
* Pointer to error structure.
*
* @return
* Action template pointer on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow_actions_template *
flow_hw_actions_template_create(struct rte_eth_dev *dev,
const struct rte_flow_actions_template_attr *attr,
const struct rte_flow_action actions[],
const struct rte_flow_action masks[],
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
int len, act_len, mask_len;
unsigned int act_num;
unsigned int i;
struct rte_flow_actions_template *at = NULL;
uint16_t pos = UINT16_MAX;
uint64_t action_flags = 0;
struct rte_flow_action tmp_action[MLX5_HW_MAX_ACTS];
struct rte_flow_action tmp_mask[MLX5_HW_MAX_ACTS];
struct rte_flow_action *ra = (void *)(uintptr_t)actions;
struct rte_flow_action *rm = (void *)(uintptr_t)masks;
int set_vlan_vid_ix = -1;
struct rte_flow_action_modify_field set_vlan_vid_spec = {0, };
struct rte_flow_action_modify_field set_vlan_vid_mask = {0, };
const struct rte_flow_action_modify_field rx_mreg = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_B,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_1,
},
.width = 32,
};
const struct rte_flow_action_modify_field rx_mreg_mask = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.width = UINT32_MAX,
};
const struct rte_flow_action rx_cpy = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &rx_mreg,
};
const struct rte_flow_action rx_cpy_mask = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &rx_mreg_mask,
};
if (mlx5_flow_hw_actions_validate(dev, attr, actions, masks,
&action_flags, error))
return NULL;
for (i = 0; ra[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
switch (ra[i].type) {
/* OF_PUSH_VLAN *MUST* come before OF_SET_VLAN_VID */
case RTE_FLOW_ACTION_TYPE_OF_PUSH_VLAN:
i += is_of_vlan_pcp_present(ra + i) ?
MLX5_HW_VLAN_PUSH_PCP_IDX :
MLX5_HW_VLAN_PUSH_VID_IDX;
break;
case RTE_FLOW_ACTION_TYPE_OF_SET_VLAN_VID:
set_vlan_vid_ix = i;
break;
default:
break;
}
}
/*
* Count flow actions to allocate required space for storing DR offsets and to check
* if temporary buffer would not be overrun.
*/
act_num = i + 1;
if (act_num >= MLX5_HW_MAX_ACTS) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, NULL, "Too many actions");
return NULL;
}
if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS &&
priv->sh->config.dv_esw_en &&
(action_flags & (MLX5_FLOW_ACTION_QUEUE | MLX5_FLOW_ACTION_RSS))) {
/* Insert META copy */
if (act_num + 1 > MLX5_HW_MAX_ACTS) {
rte_flow_error_set(error, E2BIG,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL, "cannot expand: too many actions");
return NULL;
}
/* Application should make sure only one Q/RSS exist in one rule. */
pos = flow_hw_template_expand_modify_field(actions, masks,
&rx_cpy,
&rx_cpy_mask,
tmp_action, tmp_mask,
action_flags,
act_num);
ra = tmp_action;
rm = tmp_mask;
act_num++;
action_flags |= MLX5_FLOW_ACTION_MODIFY_FIELD;
}
if (set_vlan_vid_ix != -1) {
/* If temporary action buffer was not used, copy template actions to it */
if (ra == actions && rm == masks) {
for (i = 0; i < act_num; ++i) {
tmp_action[i] = actions[i];
tmp_mask[i] = masks[i];
if (actions[i].type == RTE_FLOW_ACTION_TYPE_END)
break;
}
ra = tmp_action;
rm = tmp_mask;
}
flow_hw_set_vlan_vid(dev, ra, rm,
&set_vlan_vid_spec, &set_vlan_vid_mask,
set_vlan_vid_ix);
}
act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, ra, error);
if (act_len <= 0)
return NULL;
len = RTE_ALIGN(act_len, 16);
mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, NULL, 0, rm, error);
if (mask_len <= 0)
return NULL;
len += RTE_ALIGN(mask_len, 16);
len += RTE_ALIGN(act_num * sizeof(*at->actions_off), 16);
at = mlx5_malloc(MLX5_MEM_ZERO, len + sizeof(*at),
RTE_CACHE_LINE_SIZE, rte_socket_id());
if (!at) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate action template");
return NULL;
}
/* Actions part is in the first part. */
at->attr = *attr;
at->actions = (struct rte_flow_action *)(at + 1);
act_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->actions,
len, ra, error);
if (act_len <= 0)
goto error;
/* Masks part is in the second part. */
at->masks = (struct rte_flow_action *)(((uint8_t *)at->actions) + act_len);
mask_len = rte_flow_conv(RTE_FLOW_CONV_OP_ACTIONS, at->masks,
len - act_len, rm, error);
if (mask_len <= 0)
goto error;
/* DR actions offsets in the third part. */
at->actions_off = (uint16_t *)((uint8_t *)at->masks + mask_len);
at->actions_num = act_num;
for (i = 0; i < at->actions_num; ++i)
at->actions_off[i] = UINT16_MAX;
at->reformat_off = UINT16_MAX;
at->mhdr_off = UINT16_MAX;
at->rx_cpy_pos = pos;
/*
* mlx5 PMD hacks indirect action index directly to the action conf.
* The rte_flow_conv() function copies the content from conf pointer.
* Need to restore the indirect action index from action conf here.
*/
for (i = 0; actions->type != RTE_FLOW_ACTION_TYPE_END;
actions++, masks++, i++) {
if (actions->type == RTE_FLOW_ACTION_TYPE_INDIRECT) {
at->actions[i].conf = actions->conf;
at->masks[i].conf = masks->conf;
}
}
at->tmpl = flow_hw_dr_actions_template_create(at);
if (!at->tmpl)
goto error;
at->action_flags = action_flags;
__atomic_fetch_add(&at->refcnt, 1, __ATOMIC_RELAXED);
LIST_INSERT_HEAD(&priv->flow_hw_at, at, next);
return at;
error:
if (at) {
if (at->tmpl)
mlx5dr_action_template_destroy(at->tmpl);
mlx5_free(at);
}
return NULL;
}
/**
* Destroy flow action template.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] template
* Pointer to the action template to be destroyed.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_actions_template_destroy(struct rte_eth_dev *dev __rte_unused,
struct rte_flow_actions_template *template,
struct rte_flow_error *error __rte_unused)
{
if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
DRV_LOG(WARNING, "Action template %p is still in use.",
(void *)template);
return rte_flow_error_set(error, EBUSY,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"action template in using");
}
LIST_REMOVE(template, next);
if (template->tmpl)
mlx5dr_action_template_destroy(template->tmpl);
mlx5_free(template);
return 0;
}
static uint32_t
flow_hw_count_items(const struct rte_flow_item *items)
{
const struct rte_flow_item *curr_item;
uint32_t nb_items;
nb_items = 0;
for (curr_item = items; curr_item->type != RTE_FLOW_ITEM_TYPE_END; ++curr_item)
++nb_items;
return ++nb_items;
}
static struct rte_flow_item *
flow_hw_prepend_item(const struct rte_flow_item *items,
const uint32_t nb_items,
const struct rte_flow_item *new_item,
struct rte_flow_error *error)
{
struct rte_flow_item *copied_items;
size_t size;
/* Allocate new array of items. */
size = sizeof(*copied_items) * (nb_items + 1);
copied_items = mlx5_malloc(MLX5_MEM_ZERO, size, 0, rte_socket_id());
if (!copied_items) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate item template");
return NULL;
}
/* Put new item at the beginning and copy the rest. */
copied_items[0] = *new_item;
rte_memcpy(&copied_items[1], items, sizeof(*items) * nb_items);
return copied_items;
}
static int
flow_hw_pattern_validate(struct rte_eth_dev *dev,
const struct rte_flow_pattern_template_attr *attr,
const struct rte_flow_item items[],
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
int i;
bool items_end = false;
if (!attr->ingress && !attr->egress && !attr->transfer)
return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ATTR, NULL,
"at least one of the direction attributes"
" must be specified");
if (priv->sh->config.dv_esw_en) {
MLX5_ASSERT(priv->master || priv->representor);
if (priv->master) {
if ((attr->ingress && attr->egress) ||
(attr->ingress && attr->transfer) ||
(attr->egress && attr->transfer))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR, NULL,
"only one direction attribute at once"
" can be used on transfer proxy port");
} else {
if (attr->transfer)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
"transfer attribute cannot be used with"
" port representors");
if (attr->ingress && attr->egress)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR, NULL,
"ingress and egress direction attributes"
" cannot be used at the same time on"
" port representors");
}
} else {
if (attr->transfer)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ATTR_TRANSFER, NULL,
"transfer attribute cannot be used when"
" E-Switch is disabled");
}
for (i = 0; !items_end; i++) {
int type = items[i].type;
switch (type) {
case RTE_FLOW_ITEM_TYPE_TAG:
{
int reg;
const struct rte_flow_item_tag *tag =
(const struct rte_flow_item_tag *)items[i].spec;
reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_TAG, tag->index);
if (reg == REG_NON)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Unsupported tag index");
break;
}
case MLX5_RTE_FLOW_ITEM_TYPE_TAG:
{
const struct rte_flow_item_tag *tag =
(const struct rte_flow_item_tag *)items[i].spec;
uint8_t regcs = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;
if (!((1 << (tag->index - REG_C_0)) & regcs))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Unsupported internal tag index");
break;
}
case RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT:
if (attr->ingress && priv->sh->config.repr_matching)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM, NULL,
"represented port item cannot be used"
" when ingress attribute is set");
if (attr->egress)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM, NULL,
"represented port item cannot be used"
" when egress attribute is set");
break;
case RTE_FLOW_ITEM_TYPE_META:
if (!priv->sh->config.dv_esw_en ||
priv->sh->config.dv_xmeta_en != MLX5_XMETA_MODE_META32_HWS) {
if (attr->ingress)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ITEM, NULL,
"META item is not supported"
" on current FW with ingress"
" attribute");
}
break;
case RTE_FLOW_ITEM_TYPE_METER_COLOR:
{
int reg = flow_hw_get_reg_id(RTE_FLOW_ITEM_TYPE_METER_COLOR, 0);
if (reg == REG_NON)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Unsupported meter color register");
break;
}
case RTE_FLOW_ITEM_TYPE_VOID:
case RTE_FLOW_ITEM_TYPE_ETH:
case RTE_FLOW_ITEM_TYPE_VLAN:
case RTE_FLOW_ITEM_TYPE_IPV4:
case RTE_FLOW_ITEM_TYPE_IPV6:
case RTE_FLOW_ITEM_TYPE_UDP:
case RTE_FLOW_ITEM_TYPE_TCP:
case RTE_FLOW_ITEM_TYPE_GTP:
case RTE_FLOW_ITEM_TYPE_GTP_PSC:
case RTE_FLOW_ITEM_TYPE_VXLAN:
case MLX5_RTE_FLOW_ITEM_TYPE_SQ:
case RTE_FLOW_ITEM_TYPE_GRE:
case RTE_FLOW_ITEM_TYPE_GRE_KEY:
case RTE_FLOW_ITEM_TYPE_GRE_OPTION:
case RTE_FLOW_ITEM_TYPE_ICMP:
case RTE_FLOW_ITEM_TYPE_ICMP6:
case RTE_FLOW_ITEM_TYPE_CONNTRACK:
break;
case RTE_FLOW_ITEM_TYPE_INTEGRITY:
/*
* Integrity flow item validation require access to
* both item mask and spec.
* Current HWS model allows item mask in pattern
* template and item spec in flow rule.
*/
break;
case RTE_FLOW_ITEM_TYPE_END:
items_end = true;
break;
default:
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Unsupported item type");
}
}
return 0;
}
static bool
flow_hw_pattern_has_sq_match(const struct rte_flow_item *items)
{
unsigned int i;
for (i = 0; items[i].type != RTE_FLOW_ITEM_TYPE_END; ++i)
if (items[i].type == (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ)
return true;
return false;
}
/**
* Create flow item template.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] attr
* Pointer to the item template attributes.
* @param[in] items
* The template item pattern.
* @param[out] error
* Pointer to error structure.
*
* @return
* Item template pointer on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow_pattern_template *
flow_hw_pattern_template_create(struct rte_eth_dev *dev,
const struct rte_flow_pattern_template_attr *attr,
const struct rte_flow_item items[],
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_pattern_template *it;
struct rte_flow_item *copied_items = NULL;
const struct rte_flow_item *tmpl_items;
uint64_t orig_item_nb;
struct rte_flow_item port = {
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.mask = &rte_flow_item_ethdev_mask,
};
struct rte_flow_item_tag tag_v = {
.data = 0,
.index = REG_C_0,
};
struct rte_flow_item_tag tag_m = {
.data = flow_hw_tx_tag_regc_mask(dev),
.index = 0xff,
};
struct rte_flow_item tag = {
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
.spec = &tag_v,
.mask = &tag_m,
.last = NULL
};
if (flow_hw_pattern_validate(dev, attr, items, error))
return NULL;
orig_item_nb = flow_hw_count_items(items);
if (priv->sh->config.dv_esw_en &&
priv->sh->config.repr_matching &&
attr->ingress && !attr->egress && !attr->transfer) {
copied_items = flow_hw_prepend_item(items, orig_item_nb, &port, error);
if (!copied_items)
return NULL;
tmpl_items = copied_items;
} else if (priv->sh->config.dv_esw_en &&
priv->sh->config.repr_matching &&
!attr->ingress && attr->egress && !attr->transfer) {
if (flow_hw_pattern_has_sq_match(items)) {
DRV_LOG(DEBUG, "Port %u omitting implicit REG_C_0 match for egress "
"pattern template", dev->data->port_id);
tmpl_items = items;
goto setup_pattern_template;
}
copied_items = flow_hw_prepend_item(items, orig_item_nb, &tag, error);
if (!copied_items)
return NULL;
tmpl_items = copied_items;
} else {
tmpl_items = items;
}
setup_pattern_template:
it = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*it), 0, rte_socket_id());
if (!it) {
if (copied_items)
mlx5_free(copied_items);
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate item template");
return NULL;
}
it->attr = *attr;
it->orig_item_nb = orig_item_nb;
it->mt = mlx5dr_match_template_create(tmpl_items, attr->relaxed_matching);
if (!it->mt) {
if (copied_items)
mlx5_free(copied_items);
mlx5_free(it);
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot create match template");
return NULL;
}
it->item_flags = flow_hw_rss_item_flags_get(tmpl_items);
if (copied_items) {
if (attr->ingress)
it->implicit_port = true;
else if (attr->egress)
it->implicit_tag = true;
mlx5_free(copied_items);
}
__atomic_fetch_add(&it->refcnt, 1, __ATOMIC_RELAXED);
LIST_INSERT_HEAD(&priv->flow_hw_itt, it, next);
return it;
}
/**
* Destroy flow item template.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] template
* Pointer to the item template to be destroyed.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_pattern_template_destroy(struct rte_eth_dev *dev __rte_unused,
struct rte_flow_pattern_template *template,
struct rte_flow_error *error __rte_unused)
{
if (__atomic_load_n(&template->refcnt, __ATOMIC_RELAXED) > 1) {
DRV_LOG(WARNING, "Item template %p is still in use.",
(void *)template);
return rte_flow_error_set(error, EBUSY,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"item template in using");
}
LIST_REMOVE(template, next);
claim_zero(mlx5dr_match_template_destroy(template->mt));
mlx5_free(template);
return 0;
}
/*
* Get information about HWS pre-configurable resources.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[out] port_info
* Pointer to port information.
* @param[out] queue_info
* Pointer to queue information.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_info_get(struct rte_eth_dev *dev,
struct rte_flow_port_info *port_info,
struct rte_flow_queue_info *queue_info,
struct rte_flow_error *error __rte_unused)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint16_t port_id = dev->data->port_id;
struct rte_mtr_capabilities mtr_cap;
int ret;
memset(port_info, 0, sizeof(*port_info));
/* Queue size is unlimited from low-level. */
port_info->max_nb_queues = UINT32_MAX;
queue_info->max_size = UINT32_MAX;
memset(&mtr_cap, 0, sizeof(struct rte_mtr_capabilities));
ret = rte_mtr_capabilities_get(port_id, &mtr_cap, NULL);
if (!ret)
port_info->max_nb_meters = mtr_cap.n_max;
port_info->max_nb_counters = priv->sh->hws_max_nb_counters;
port_info->max_nb_aging_objects = port_info->max_nb_counters;
return 0;
}
/**
* Create group callback.
*
* @param[in] tool_ctx
* Pointer to the hash list related context.
* @param[in] cb_ctx
* Pointer to the group creation context.
*
* @return
* Group entry on success, NULL otherwise and rte_errno is set.
*/
struct mlx5_list_entry *
flow_hw_grp_create_cb(void *tool_ctx, void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
struct rte_eth_dev *dev = ctx->dev;
struct rte_flow_attr *attr = (struct rte_flow_attr *)ctx->data;
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5dr_table_attr dr_tbl_attr = {0};
struct rte_flow_error *error = ctx->error;
struct mlx5_flow_group *grp_data;
struct mlx5dr_table *tbl = NULL;
struct mlx5dr_action *jump;
uint32_t idx = 0;
grp_data = mlx5_ipool_zmalloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
if (!grp_data) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate flow table data entry");
return NULL;
}
dr_tbl_attr.level = attr->group;
if (attr->transfer)
dr_tbl_attr.type = MLX5DR_TABLE_TYPE_FDB;
else if (attr->egress)
dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_TX;
else
dr_tbl_attr.type = MLX5DR_TABLE_TYPE_NIC_RX;
tbl = mlx5dr_table_create(priv->dr_ctx, &dr_tbl_attr);
if (!tbl)
goto error;
grp_data->tbl = tbl;
if (attr->group) {
/* Jump action be used by non-root table. */
jump = mlx5dr_action_create_dest_table
(priv->dr_ctx, tbl,
mlx5_hw_act_flag[!!attr->group][dr_tbl_attr.type]);
if (!jump)
goto error;
grp_data->jump.hws_action = jump;
/* Jump action be used by root table. */
jump = mlx5dr_action_create_dest_table
(priv->dr_ctx, tbl,
mlx5_hw_act_flag[MLX5_HW_ACTION_FLAG_ROOT]
[dr_tbl_attr.type]);
if (!jump)
goto error;
grp_data->jump.root_action = jump;
}
grp_data->dev = dev;
grp_data->idx = idx;
grp_data->group_id = attr->group;
grp_data->type = dr_tbl_attr.type;
return &grp_data->entry;
error:
if (grp_data->jump.root_action)
mlx5dr_action_destroy(grp_data->jump.root_action);
if (grp_data->jump.hws_action)
mlx5dr_action_destroy(grp_data->jump.hws_action);
if (tbl)
mlx5dr_table_destroy(tbl);
if (idx)
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], idx);
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate flow dr table");
return NULL;
}
/**
* Remove group callback.
*
* @param[in] tool_ctx
* Pointer to the hash list related context.
* @param[in] entry
* Pointer to the entry to be removed.
*/
void
flow_hw_grp_remove_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_group *grp_data =
container_of(entry, struct mlx5_flow_group, entry);
MLX5_ASSERT(entry && sh);
/* To use the wrapper glue functions instead. */
if (grp_data->jump.hws_action)
mlx5dr_action_destroy(grp_data->jump.hws_action);
if (grp_data->jump.root_action)
mlx5dr_action_destroy(grp_data->jump.root_action);
mlx5dr_table_destroy(grp_data->tbl);
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}
/**
* Match group callback.
*
* @param[in] tool_ctx
* Pointer to the hash list related context.
* @param[in] entry
* Pointer to the group to be matched.
* @param[in] cb_ctx
* Pointer to the group matching context.
*
* @return
* 0 on matched, 1 on miss matched.
*/
int
flow_hw_grp_match_cb(void *tool_ctx __rte_unused, struct mlx5_list_entry *entry,
void *cb_ctx)
{
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
struct mlx5_flow_group *grp_data =
container_of(entry, struct mlx5_flow_group, entry);
struct rte_flow_attr *attr =
(struct rte_flow_attr *)ctx->data;
return (grp_data->dev != ctx->dev) ||
(grp_data->group_id != attr->group) ||
((grp_data->type != MLX5DR_TABLE_TYPE_FDB) &&
attr->transfer) ||
((grp_data->type != MLX5DR_TABLE_TYPE_NIC_TX) &&
attr->egress) ||
((grp_data->type != MLX5DR_TABLE_TYPE_NIC_RX) &&
attr->ingress);
}
/**
* Clone group entry callback.
*
* @param[in] tool_ctx
* Pointer to the hash list related context.
* @param[in] entry
* Pointer to the group to be matched.
* @param[in] cb_ctx
* Pointer to the group matching context.
*
* @return
* 0 on matched, 1 on miss matched.
*/
struct mlx5_list_entry *
flow_hw_grp_clone_cb(void *tool_ctx, struct mlx5_list_entry *oentry,
void *cb_ctx)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_cb_ctx *ctx = cb_ctx;
struct mlx5_flow_group *grp_data;
struct rte_flow_error *error = ctx->error;
uint32_t idx = 0;
grp_data = mlx5_ipool_malloc(sh->ipool[MLX5_IPOOL_HW_GRP], &idx);
if (!grp_data) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"cannot allocate flow table data entry");
return NULL;
}
memcpy(grp_data, oentry, sizeof(*grp_data));
grp_data->idx = idx;
return &grp_data->entry;
}
/**
* Free cloned group entry callback.
*
* @param[in] tool_ctx
* Pointer to the hash list related context.
* @param[in] entry
* Pointer to the group to be freed.
*/
void
flow_hw_grp_clone_free_cb(void *tool_ctx, struct mlx5_list_entry *entry)
{
struct mlx5_dev_ctx_shared *sh = tool_ctx;
struct mlx5_flow_group *grp_data =
container_of(entry, struct mlx5_flow_group, entry);
mlx5_ipool_free(sh->ipool[MLX5_IPOOL_HW_GRP], grp_data->idx);
}
/**
* Create and cache a vport action for given @p dev port. vport actions
* cache is used in HWS with FDB flows.
*
* This function does not create any function if proxy port for @p dev port
* was not configured for HW Steering.
*
* This function assumes that E-Switch is enabled and PMD is running with
* HW Steering configured.
*
* @param dev
* Pointer to Ethernet device which will be the action destination.
*
* @return
* 0 on success, positive value otherwise.
*/
int
flow_hw_create_vport_action(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_eth_dev *proxy_dev;
struct mlx5_priv *proxy_priv;
uint16_t port_id = dev->data->port_id;
uint16_t proxy_port_id = port_id;
int ret;
ret = mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL);
if (ret)
return ret;
proxy_dev = &rte_eth_devices[proxy_port_id];
proxy_priv = proxy_dev->data->dev_private;
if (!proxy_priv->hw_vport)
return 0;
if (proxy_priv->hw_vport[port_id]) {
DRV_LOG(ERR, "port %u HWS vport action already created",
port_id);
return -EINVAL;
}
proxy_priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
(proxy_priv->dr_ctx, priv->dev_port,
MLX5DR_ACTION_FLAG_HWS_FDB);
if (!proxy_priv->hw_vport[port_id]) {
DRV_LOG(ERR, "port %u unable to create HWS vport action",
port_id);
return -EINVAL;
}
return 0;
}
/**
* Destroys the vport action associated with @p dev device
* from actions' cache.
*
* This function does not destroy any action if there is no action cached
* for @p dev or proxy port was not configured for HW Steering.
*
* This function assumes that E-Switch is enabled and PMD is running with
* HW Steering configured.
*
* @param dev
* Pointer to Ethernet device which will be the action destination.
*/
void
flow_hw_destroy_vport_action(struct rte_eth_dev *dev)
{
struct rte_eth_dev *proxy_dev;
struct mlx5_priv *proxy_priv;
uint16_t port_id = dev->data->port_id;
uint16_t proxy_port_id = port_id;
if (mlx5_flow_pick_transfer_proxy(dev, &proxy_port_id, NULL))
return;
proxy_dev = &rte_eth_devices[proxy_port_id];
proxy_priv = proxy_dev->data->dev_private;
if (!proxy_priv->hw_vport || !proxy_priv->hw_vport[port_id])
return;
mlx5dr_action_destroy(proxy_priv->hw_vport[port_id]);
proxy_priv->hw_vport[port_id] = NULL;
}
static int
flow_hw_create_vport_actions(struct mlx5_priv *priv)
{
uint16_t port_id;
MLX5_ASSERT(!priv->hw_vport);
priv->hw_vport = mlx5_malloc(MLX5_MEM_ZERO,
sizeof(*priv->hw_vport) * RTE_MAX_ETHPORTS,
0, SOCKET_ID_ANY);
if (!priv->hw_vport)
return -ENOMEM;
DRV_LOG(DEBUG, "port %u :: creating vport actions", priv->dev_data->port_id);
DRV_LOG(DEBUG, "port %u :: domain_id=%u", priv->dev_data->port_id, priv->domain_id);
MLX5_ETH_FOREACH_DEV(port_id, NULL) {
struct mlx5_priv *port_priv = rte_eth_devices[port_id].data->dev_private;
if (!port_priv ||
port_priv->domain_id != priv->domain_id)
continue;
DRV_LOG(DEBUG, "port %u :: for port_id=%u, calling mlx5dr_action_create_dest_vport() with ibport=%u",
priv->dev_data->port_id, port_id, port_priv->dev_port);
priv->hw_vport[port_id] = mlx5dr_action_create_dest_vport
(priv->dr_ctx, port_priv->dev_port,
MLX5DR_ACTION_FLAG_HWS_FDB);
DRV_LOG(DEBUG, "port %u :: priv->hw_vport[%u]=%p",
priv->dev_data->port_id, port_id, (void *)priv->hw_vport[port_id]);
if (!priv->hw_vport[port_id])
return -EINVAL;
}
return 0;
}
static void
flow_hw_free_vport_actions(struct mlx5_priv *priv)
{
uint16_t port_id;
if (!priv->hw_vport)
return;
for (port_id = 0; port_id < RTE_MAX_ETHPORTS; ++port_id)
if (priv->hw_vport[port_id])
mlx5dr_action_destroy(priv->hw_vport[port_id]);
mlx5_free(priv->hw_vport);
priv->hw_vport = NULL;
}
/**
* Create an egress pattern template matching on source SQ.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to pattern template on success. NULL otherwise, and rte_errno is set.
*/
static struct rte_flow_pattern_template *
flow_hw_create_tx_repr_sq_pattern_tmpl(struct rte_eth_dev *dev)
{
struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.egress = 1,
};
struct mlx5_rte_flow_item_sq sq_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.mask = &sq_mask,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_mask(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t mask = priv->sh->dv_regc0_mask;
/* Mask is verified during device initialization. Sanity checking here. */
MLX5_ASSERT(mask != 0);
/*
* Availability of sufficient number of bits in REG_C_0 is verified on initialization.
* Sanity checking here.
*/
MLX5_ASSERT(__builtin_popcount(mask) >= __builtin_popcount(priv->vport_meta_mask));
return mask;
}
static __rte_always_inline uint32_t
flow_hw_tx_tag_regc_value(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t tag;
/* Mask is verified during device initialization. Sanity checking here. */
MLX5_ASSERT(priv->vport_meta_mask != 0);
tag = priv->vport_meta_tag >> (rte_bsf32(priv->vport_meta_mask));
/*
* Availability of sufficient number of bits in REG_C_0 is verified on initialization.
* Sanity checking here.
*/
MLX5_ASSERT((tag & priv->sh->dv_regc0_mask) == tag);
return tag;
}
static void
flow_hw_update_action_mask(struct rte_flow_action *action,
struct rte_flow_action *mask,
enum rte_flow_action_type type,
void *conf_v,
void *conf_m)
{
action->type = type;
action->conf = conf_v;
mask->type = type;
mask->conf = conf_m;
}
/**
* Create an egress actions template with MODIFY_FIELD action for setting unused REG_C_0 bits
* to vport tag and JUMP action to group 1.
*
* If extended metadata mode is enabled, then MODIFY_FIELD action for copying software metadata
* to REG_C_1 is added as well.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to actions template on success. NULL otherwise, and rte_errno is set.
*/
static struct rte_flow_actions_template *
flow_hw_create_tx_repr_tag_jump_acts_tmpl(struct rte_eth_dev *dev)
{
uint32_t tag_mask = flow_hw_tx_tag_regc_mask(dev);
uint32_t tag_value = flow_hw_tx_tag_regc_value(dev);
struct rte_flow_actions_template_attr attr = {
.egress = 1,
};
struct rte_flow_action_modify_field set_tag_v = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_0,
.offset = rte_bsf32(tag_mask),
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
},
.width = __builtin_popcount(tag_mask),
};
struct rte_flow_action_modify_field set_tag_m = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
},
.width = UINT32_MAX,
};
struct rte_flow_action_modify_field copy_metadata_v = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_1,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_A,
},
.width = 32,
};
struct rte_flow_action_modify_field copy_metadata_m = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.width = UINT32_MAX,
};
struct rte_flow_action_jump jump_v = {
.group = MLX5_HW_LOWEST_USABLE_GROUP,
};
struct rte_flow_action_jump jump_m = {
.group = UINT32_MAX,
};
struct rte_flow_action actions_v[4] = { { 0 } };
struct rte_flow_action actions_m[4] = { { 0 } };
unsigned int idx = 0;
rte_memcpy(set_tag_v.src.value, &tag_value, sizeof(tag_value));
rte_memcpy(set_tag_m.src.value, &tag_mask, sizeof(tag_mask));
flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
&set_tag_v, &set_tag_m);
idx++;
if (MLX5_SH(dev)->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx],
RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
&copy_metadata_v, &copy_metadata_m);
idx++;
}
flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_JUMP,
&jump_v, &jump_m);
idx++;
flow_hw_update_action_mask(&actions_v[idx], &actions_m[idx], RTE_FLOW_ACTION_TYPE_END,
NULL, NULL);
idx++;
MLX5_ASSERT(idx <= RTE_DIM(actions_v));
return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL);
}
static void
flow_hw_cleanup_tx_repr_tagging(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
if (priv->hw_tx_repr_tagging_tbl) {
flow_hw_table_destroy(dev, priv->hw_tx_repr_tagging_tbl, NULL);
priv->hw_tx_repr_tagging_tbl = NULL;
}
if (priv->hw_tx_repr_tagging_at) {
flow_hw_actions_template_destroy(dev, priv->hw_tx_repr_tagging_at, NULL);
priv->hw_tx_repr_tagging_at = NULL;
}
if (priv->hw_tx_repr_tagging_pt) {
flow_hw_pattern_template_destroy(dev, priv->hw_tx_repr_tagging_pt, NULL);
priv->hw_tx_repr_tagging_pt = NULL;
}
}
/**
* Setup templates and table used to create default Tx flow rules. These default rules
* allow for matching Tx representor traffic using a vport tag placed in unused bits of
* REG_C_0 register.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, negative errno value otherwise.
*/
static int
flow_hw_setup_tx_repr_tagging(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_template_table_attr attr = {
.flow_attr = {
.group = 0,
.priority = MLX5_HW_LOWEST_PRIO_ROOT,
.egress = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
struct mlx5_flow_template_table_cfg cfg = {
.attr = attr,
.external = false,
};
MLX5_ASSERT(priv->sh->config.dv_esw_en);
MLX5_ASSERT(priv->sh->config.repr_matching);
priv->hw_tx_repr_tagging_pt = flow_hw_create_tx_repr_sq_pattern_tmpl(dev);
if (!priv->hw_tx_repr_tagging_pt)
goto error;
priv->hw_tx_repr_tagging_at = flow_hw_create_tx_repr_tag_jump_acts_tmpl(dev);
if (!priv->hw_tx_repr_tagging_at)
goto error;
priv->hw_tx_repr_tagging_tbl = flow_hw_table_create(dev, &cfg,
&priv->hw_tx_repr_tagging_pt, 1,
&priv->hw_tx_repr_tagging_at, 1,
NULL);
if (!priv->hw_tx_repr_tagging_tbl)
goto error;
return 0;
error:
flow_hw_cleanup_tx_repr_tagging(dev);
return -rte_errno;
}
static uint32_t
flow_hw_esw_mgr_regc_marker_mask(struct rte_eth_dev *dev)
{
uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;
/* Mask is verified during device initialization. */
MLX5_ASSERT(mask != 0);
return mask;
}
static uint32_t
flow_hw_esw_mgr_regc_marker(struct rte_eth_dev *dev)
{
uint32_t mask = MLX5_SH(dev)->dv_regc0_mask;
/* Mask is verified during device initialization. */
MLX5_ASSERT(mask != 0);
return RTE_BIT32(rte_bsf32(mask));
}
/**
* Creates a flow pattern template used to match on E-Switch Manager.
* This template is used to set up a table for SQ miss default flow.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow pattern template on success, NULL otherwise.
*/
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_esw_mgr_pattern_template(struct rte_eth_dev *dev)
{
struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.transfer = 1,
};
struct rte_flow_item_ethdev port_spec = {
.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
};
struct rte_flow_item_ethdev port_mask = {
.port_id = UINT16_MAX,
};
struct mlx5_rte_flow_item_sq sq_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.spec = &port_spec,
.mask = &port_mask,
},
{
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.mask = &sq_mask,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
/**
* Creates a flow pattern template used to match REG_C_0 and a SQ.
* Matching on REG_C_0 is set up to match on all bits usable by user-space.
* If traffic was sent from E-Switch Manager, then all usable bits will be set to 0,
* except the least significant bit, which will be set to 1.
*
* This template is used to set up a table for SQ miss default flow.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow pattern template on success, NULL otherwise.
*/
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_regc_sq_pattern_template(struct rte_eth_dev *dev)
{
struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.transfer = 1,
};
struct rte_flow_item_tag reg_c0_spec = {
.index = (uint8_t)REG_C_0,
};
struct rte_flow_item_tag reg_c0_mask = {
.index = 0xff,
.data = flow_hw_esw_mgr_regc_marker_mask(dev),
};
struct mlx5_rte_flow_item_sq queue_mask = {
.queue = UINT32_MAX,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)
MLX5_RTE_FLOW_ITEM_TYPE_TAG,
.spec = &reg_c0_spec,
.mask = &reg_c0_mask,
},
{
.type = (enum rte_flow_item_type)
MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.mask = &queue_mask,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
/**
* Creates a flow pattern template with unmasked represented port matching.
* This template is used to set up a table for default transfer flows
* directing packets to group 1.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow pattern template on success, NULL otherwise.
*/
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_port_pattern_template(struct rte_eth_dev *dev)
{
struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.transfer = 1,
};
struct rte_flow_item_ethdev port_mask = {
.port_id = UINT16_MAX,
};
struct rte_flow_item items[] = {
{
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.mask = &port_mask,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
/*
* Creating a flow pattern template with all ETH packets matching.
* This template is used to set up a table for default Tx copy (Tx metadata
* to REG_C_1) flow rule usage.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow pattern template on success, NULL otherwise.
*/
static struct rte_flow_pattern_template *
flow_hw_create_tx_default_mreg_copy_pattern_template(struct rte_eth_dev *dev)
{
struct rte_flow_pattern_template_attr tx_pa_attr = {
.relaxed_matching = 0,
.egress = 1,
};
struct rte_flow_item_eth promisc = {
.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
struct rte_flow_item eth_all[] = {
[0] = {
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = &promisc,
.mask = &promisc,
},
[1] = {
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
struct rte_flow_error drop_err;
RTE_SET_USED(drop_err);
return flow_hw_pattern_template_create(dev, &tx_pa_attr, eth_all, &drop_err);
}
/**
* Creates a flow actions template with modify field action and masked jump action.
* Modify field action sets the least significant bit of REG_C_0 (usable by user-space)
* to 1, meaning that packet was originated from E-Switch Manager. Jump action
* transfers steering to group 1.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow actions template on success, NULL otherwise.
*/
static struct rte_flow_actions_template *
flow_hw_create_ctrl_regc_jump_actions_template(struct rte_eth_dev *dev)
{
uint32_t marker_mask = flow_hw_esw_mgr_regc_marker_mask(dev);
uint32_t marker_bits = flow_hw_esw_mgr_regc_marker(dev);
struct rte_flow_actions_template_attr attr = {
.transfer = 1,
};
struct rte_flow_action_modify_field set_reg_v = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_0,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
},
.width = __builtin_popcount(marker_mask),
};
struct rte_flow_action_modify_field set_reg_m = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.src = {
.field = RTE_FLOW_FIELD_VALUE,
},
.width = UINT32_MAX,
};
struct rte_flow_action_jump jump_v = {
.group = MLX5_HW_LOWEST_USABLE_GROUP,
};
struct rte_flow_action_jump jump_m = {
.group = UINT32_MAX,
};
struct rte_flow_action actions_v[] = {
{
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &set_reg_v,
},
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_v,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_flow_action actions_m[] = {
{
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &set_reg_m,
},
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_m,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
set_reg_v.dst.offset = rte_bsf32(marker_mask);
rte_memcpy(set_reg_v.src.value, &marker_bits, sizeof(marker_bits));
rte_memcpy(set_reg_m.src.value, &marker_mask, sizeof(marker_mask));
return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m, NULL);
}
/**
* Creates a flow actions template with an unmasked JUMP action. Flows
* based on this template will perform a jump to some group. This template
* is used to set up tables for control flows.
*
* @param dev
* Pointer to Ethernet device.
* @param group
* Destination group for this action template.
*
* @return
* Pointer to flow actions template on success, NULL otherwise.
*/
static struct rte_flow_actions_template *
flow_hw_create_ctrl_jump_actions_template(struct rte_eth_dev *dev,
uint32_t group)
{
struct rte_flow_actions_template_attr attr = {
.transfer = 1,
};
struct rte_flow_action_jump jump_v = {
.group = group,
};
struct rte_flow_action_jump jump_m = {
.group = UINT32_MAX,
};
struct rte_flow_action actions_v[] = {
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_v,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_flow_action actions_m[] = {
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_m,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m,
NULL);
}
/**
* Creates a flow action template with a unmasked REPRESENTED_PORT action.
* It is used to create control flow tables.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow action template on success, NULL otherwise.
*/
static struct rte_flow_actions_template *
flow_hw_create_ctrl_port_actions_template(struct rte_eth_dev *dev)
{
struct rte_flow_actions_template_attr attr = {
.transfer = 1,
};
struct rte_flow_action_ethdev port_v = {
.port_id = 0,
};
struct rte_flow_action actions_v[] = {
{
.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
.conf = &port_v,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_flow_action_ethdev port_m = {
.port_id = 0,
};
struct rte_flow_action actions_m[] = {
{
.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
.conf = &port_m,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
return flow_hw_actions_template_create(dev, &attr, actions_v, actions_m,
NULL);
}
/*
* Creating an actions template to use header modify action for register
* copying. This template is used to set up a table for copy flow.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* Pointer to flow actions template on success, NULL otherwise.
*/
static struct rte_flow_actions_template *
flow_hw_create_tx_default_mreg_copy_actions_template(struct rte_eth_dev *dev)
{
struct rte_flow_actions_template_attr tx_act_attr = {
.egress = 1,
};
const struct rte_flow_action_modify_field mreg_action = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_1,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_A,
},
.width = 32,
};
const struct rte_flow_action_modify_field mreg_mask = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = UINT32_MAX,
.offset = UINT32_MAX,
},
.width = UINT32_MAX,
};
const struct rte_flow_action_jump jump_action = {
.group = 1,
};
const struct rte_flow_action_jump jump_mask = {
.group = UINT32_MAX,
};
const struct rte_flow_action actions[] = {
[0] = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &mreg_action,
},
[1] = {
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_action,
},
[2] = {
.type = RTE_FLOW_ACTION_TYPE_END,
},
};
const struct rte_flow_action masks[] = {
[0] = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &mreg_mask,
},
[1] = {
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump_mask,
},
[2] = {
.type = RTE_FLOW_ACTION_TYPE_END,
},
};
struct rte_flow_error drop_err;
RTE_SET_USED(drop_err);
return flow_hw_actions_template_create(dev, &tx_act_attr, actions,
masks, &drop_err);
}
/**
* Creates a control flow table used to transfer traffic from E-Switch Manager
* and TX queues from group 0 to group 1.
*
* @param dev
* Pointer to Ethernet device.
* @param it
* Pointer to flow pattern template.
* @param at
* Pointer to flow actions template.
*
* @return
* Pointer to flow table on success, NULL otherwise.
*/
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_root_table(struct rte_eth_dev *dev,
struct rte_flow_pattern_template *it,
struct rte_flow_actions_template *at)
{
struct rte_flow_template_table_attr attr = {
.flow_attr = {
.group = 0,
.priority = MLX5_HW_LOWEST_PRIO_ROOT,
.ingress = 0,
.egress = 0,
.transfer = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
struct mlx5_flow_template_table_cfg cfg = {
.attr = attr,
.external = false,
};
return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}
/**
* Creates a control flow table used to transfer traffic from E-Switch Manager
* and TX queues from group 0 to group 1.
*
* @param dev
* Pointer to Ethernet device.
* @param it
* Pointer to flow pattern template.
* @param at
* Pointer to flow actions template.
*
* @return
* Pointer to flow table on success, NULL otherwise.
*/
static struct rte_flow_template_table*
flow_hw_create_ctrl_sq_miss_table(struct rte_eth_dev *dev,
struct rte_flow_pattern_template *it,
struct rte_flow_actions_template *at)
{
struct rte_flow_template_table_attr attr = {
.flow_attr = {
.group = 1,
.priority = MLX5_HW_LOWEST_PRIO_NON_ROOT,
.ingress = 0,
.egress = 0,
.transfer = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
struct mlx5_flow_template_table_cfg cfg = {
.attr = attr,
.external = false,
};
return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}
/*
* Creating the default Tx metadata copy table on NIC Tx group 0.
*
* @param dev
* Pointer to Ethernet device.
* @param pt
* Pointer to flow pattern template.
* @param at
* Pointer to flow actions template.
*
* @return
* Pointer to flow table on success, NULL otherwise.
*/
static struct rte_flow_template_table*
flow_hw_create_tx_default_mreg_copy_table(struct rte_eth_dev *dev,
struct rte_flow_pattern_template *pt,
struct rte_flow_actions_template *at)
{
struct rte_flow_template_table_attr tx_tbl_attr = {
.flow_attr = {
.group = 0, /* Root */
.priority = MLX5_HW_LOWEST_PRIO_ROOT,
.egress = 1,
},
.nb_flows = 1, /* One default flow rule for all. */
};
struct mlx5_flow_template_table_cfg tx_tbl_cfg = {
.attr = tx_tbl_attr,
.external = false,
};
struct rte_flow_error drop_err = {
.type = RTE_FLOW_ERROR_TYPE_NONE,
.cause = NULL,
.message = NULL,
};
RTE_SET_USED(drop_err);
return flow_hw_table_create(dev, &tx_tbl_cfg, &pt, 1, &at, 1, &drop_err);
}
/**
* Creates a control flow table used to transfer traffic
* from group 0 to group 1.
*
* @param dev
* Pointer to Ethernet device.
* @param it
* Pointer to flow pattern template.
* @param at
* Pointer to flow actions template.
*
* @return
* Pointer to flow table on success, NULL otherwise.
*/
static struct rte_flow_template_table *
flow_hw_create_ctrl_jump_table(struct rte_eth_dev *dev,
struct rte_flow_pattern_template *it,
struct rte_flow_actions_template *at)
{
struct rte_flow_template_table_attr attr = {
.flow_attr = {
.group = 0,
.priority = 0,
.ingress = 0,
.egress = 0,
.transfer = 1,
},
.nb_flows = MLX5_HW_CTRL_FLOW_NB_RULES,
};
struct mlx5_flow_template_table_cfg cfg = {
.attr = attr,
.external = false,
};
return flow_hw_table_create(dev, &cfg, &it, 1, &at, 1, NULL);
}
/**
* Creates a set of flow tables used to create control flows used
* when E-Switch is engaged.
*
* @param dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, EINVAL otherwise
*/
static __rte_unused int
flow_hw_create_ctrl_tables(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_pattern_template *esw_mgr_items_tmpl = NULL;
struct rte_flow_pattern_template *regc_sq_items_tmpl = NULL;
struct rte_flow_pattern_template *port_items_tmpl = NULL;
struct rte_flow_pattern_template *tx_meta_items_tmpl = NULL;
struct rte_flow_actions_template *regc_jump_actions_tmpl = NULL;
struct rte_flow_actions_template *port_actions_tmpl = NULL;
struct rte_flow_actions_template *jump_one_actions_tmpl = NULL;
struct rte_flow_actions_template *tx_meta_actions_tmpl = NULL;
uint32_t xmeta = priv->sh->config.dv_xmeta_en;
uint32_t repr_matching = priv->sh->config.repr_matching;
/* Create templates and table for default SQ miss flow rules - root table. */
esw_mgr_items_tmpl = flow_hw_create_ctrl_esw_mgr_pattern_template(dev);
if (!esw_mgr_items_tmpl) {
DRV_LOG(ERR, "port %u failed to create E-Switch Manager item"
" template for control flows", dev->data->port_id);
goto error;
}
regc_jump_actions_tmpl = flow_hw_create_ctrl_regc_jump_actions_template(dev);
if (!regc_jump_actions_tmpl) {
DRV_LOG(ERR, "port %u failed to create REG_C set and jump action template"
" for control flows", dev->data->port_id);
goto error;
}
MLX5_ASSERT(priv->hw_esw_sq_miss_root_tbl == NULL);
priv->hw_esw_sq_miss_root_tbl = flow_hw_create_ctrl_sq_miss_root_table
(dev, esw_mgr_items_tmpl, regc_jump_actions_tmpl);
if (!priv->hw_esw_sq_miss_root_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default sq miss (root table)"
" for control flows", dev->data->port_id);
goto error;
}
/* Create templates and table for default SQ miss flow rules - non-root table. */
regc_sq_items_tmpl = flow_hw_create_ctrl_regc_sq_pattern_template(dev);
if (!regc_sq_items_tmpl) {
DRV_LOG(ERR, "port %u failed to create SQ item template for"
" control flows", dev->data->port_id);
goto error;
}
port_actions_tmpl = flow_hw_create_ctrl_port_actions_template(dev);
if (!port_actions_tmpl) {
DRV_LOG(ERR, "port %u failed to create port action template"
" for control flows", dev->data->port_id);
goto error;
}
MLX5_ASSERT(priv->hw_esw_sq_miss_tbl == NULL);
priv->hw_esw_sq_miss_tbl = flow_hw_create_ctrl_sq_miss_table(dev, regc_sq_items_tmpl,
port_actions_tmpl);
if (!priv->hw_esw_sq_miss_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default sq miss (non-root table)"
" for control flows", dev->data->port_id);
goto error;
}
/* Create templates and table for default FDB jump flow rules. */
port_items_tmpl = flow_hw_create_ctrl_port_pattern_template(dev);
if (!port_items_tmpl) {
DRV_LOG(ERR, "port %u failed to create SQ item template for"
" control flows", dev->data->port_id);
goto error;
}
jump_one_actions_tmpl = flow_hw_create_ctrl_jump_actions_template
(dev, MLX5_HW_LOWEST_USABLE_GROUP);
if (!jump_one_actions_tmpl) {
DRV_LOG(ERR, "port %u failed to create jump action template"
" for control flows", dev->data->port_id);
goto error;
}
MLX5_ASSERT(priv->hw_esw_zero_tbl == NULL);
priv->hw_esw_zero_tbl = flow_hw_create_ctrl_jump_table(dev, port_items_tmpl,
jump_one_actions_tmpl);
if (!priv->hw_esw_zero_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default jump to group 1"
" for control flows", dev->data->port_id);
goto error;
}
/* Create templates and table for default Tx metadata copy flow rule. */
if (!repr_matching && xmeta == MLX5_XMETA_MODE_META32_HWS) {
tx_meta_items_tmpl = flow_hw_create_tx_default_mreg_copy_pattern_template(dev);
if (!tx_meta_items_tmpl) {
DRV_LOG(ERR, "port %u failed to Tx metadata copy pattern"
" template for control flows", dev->data->port_id);
goto error;
}
tx_meta_actions_tmpl = flow_hw_create_tx_default_mreg_copy_actions_template(dev);
if (!tx_meta_actions_tmpl) {
DRV_LOG(ERR, "port %u failed to Tx metadata copy actions"
" template for control flows", dev->data->port_id);
goto error;
}
MLX5_ASSERT(priv->hw_tx_meta_cpy_tbl == NULL);
priv->hw_tx_meta_cpy_tbl = flow_hw_create_tx_default_mreg_copy_table(dev,
tx_meta_items_tmpl, tx_meta_actions_tmpl);
if (!priv->hw_tx_meta_cpy_tbl) {
DRV_LOG(ERR, "port %u failed to create table for default"
" Tx metadata copy flow rule", dev->data->port_id);
goto error;
}
}
return 0;
error:
if (priv->hw_esw_zero_tbl) {
flow_hw_table_destroy(dev, priv->hw_esw_zero_tbl, NULL);
priv->hw_esw_zero_tbl = NULL;
}
if (priv->hw_esw_sq_miss_tbl) {
flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_tbl, NULL);
priv->hw_esw_sq_miss_tbl = NULL;
}
if (priv->hw_esw_sq_miss_root_tbl) {
flow_hw_table_destroy(dev, priv->hw_esw_sq_miss_root_tbl, NULL);
priv->hw_esw_sq_miss_root_tbl = NULL;
}
if (tx_meta_actions_tmpl)
flow_hw_actions_template_destroy(dev, tx_meta_actions_tmpl, NULL);
if (jump_one_actions_tmpl)
flow_hw_actions_template_destroy(dev, jump_one_actions_tmpl, NULL);
if (port_actions_tmpl)
flow_hw_actions_template_destroy(dev, port_actions_tmpl, NULL);
if (regc_jump_actions_tmpl)
flow_hw_actions_template_destroy(dev, regc_jump_actions_tmpl, NULL);
if (tx_meta_items_tmpl)
flow_hw_pattern_template_destroy(dev, tx_meta_items_tmpl, NULL);
if (port_items_tmpl)
flow_hw_pattern_template_destroy(dev, port_items_tmpl, NULL);
if (regc_sq_items_tmpl)
flow_hw_pattern_template_destroy(dev, regc_sq_items_tmpl, NULL);
if (esw_mgr_items_tmpl)
flow_hw_pattern_template_destroy(dev, esw_mgr_items_tmpl, NULL);
return -EINVAL;
}
static void
flow_hw_ct_mng_destroy(struct rte_eth_dev *dev,
struct mlx5_aso_ct_pools_mng *ct_mng)
{
struct mlx5_priv *priv = dev->data->dev_private;
mlx5_aso_ct_queue_uninit(priv->sh, ct_mng);
mlx5_free(ct_mng);
}
static void
flow_hw_ct_pool_destroy(struct rte_eth_dev *dev __rte_unused,
struct mlx5_aso_ct_pool *pool)
{
if (pool->dr_action)
mlx5dr_action_destroy(pool->dr_action);
if (pool->devx_obj)
claim_zero(mlx5_devx_cmd_destroy(pool->devx_obj));
if (pool->cts)
mlx5_ipool_destroy(pool->cts);
mlx5_free(pool);
}
static struct mlx5_aso_ct_pool *
flow_hw_ct_pool_create(struct rte_eth_dev *dev,
const struct rte_flow_port_attr *port_attr)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_pool *pool;
struct mlx5_devx_obj *obj;
uint32_t nb_cts = rte_align32pow2(port_attr->nb_conn_tracks);
uint32_t log_obj_size = rte_log2_u32(nb_cts);
struct mlx5_indexed_pool_config cfg = {
.size = sizeof(struct mlx5_aso_ct_action),
.trunk_size = 1 << 12,
.per_core_cache = 1 << 13,
.need_lock = 1,
.release_mem_en = !!priv->sh->config.reclaim_mode,
.malloc = mlx5_malloc,
.free = mlx5_free,
.type = "mlx5_hw_ct_action",
};
int reg_id;
uint32_t flags;
pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool), 0, SOCKET_ID_ANY);
if (!pool) {
rte_errno = ENOMEM;
return NULL;
}
obj = mlx5_devx_cmd_create_conn_track_offload_obj(priv->sh->cdev->ctx,
priv->sh->cdev->pdn,
log_obj_size);
if (!obj) {
rte_errno = ENODATA;
DRV_LOG(ERR, "Failed to create conn_track_offload_obj using DevX.");
goto err;
}
pool->devx_obj = obj;
reg_id = mlx5_flow_get_reg_id(dev, MLX5_ASO_CONNTRACK, 0, NULL);
flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
if (priv->sh->config.dv_esw_en && priv->master)
flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
pool->dr_action = mlx5dr_action_create_aso_ct(priv->dr_ctx,
(struct mlx5dr_devx_obj *)obj,
reg_id - REG_C_0, flags);
if (!pool->dr_action)
goto err;
/*
* No need for local cache if CT number is a small number. Since
* flow insertion rate will be very limited in that case. Here let's
* set the number to less than default trunk size 4K.
*/
if (nb_cts <= cfg.trunk_size) {
cfg.per_core_cache = 0;
cfg.trunk_size = nb_cts;
} else if (nb_cts <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
}
pool->cts = mlx5_ipool_create(&cfg);
if (!pool->cts)
goto err;
pool->sq = priv->ct_mng->aso_sqs;
/* Assign the last extra ASO SQ as public SQ. */
pool->shared_sq = &priv->ct_mng->aso_sqs[priv->nb_queue - 1];
return pool;
err:
flow_hw_ct_pool_destroy(dev, pool);
return NULL;
}
static void
flow_hw_destroy_vlan(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
enum mlx5dr_table_type i;
for (i = MLX5DR_TABLE_TYPE_NIC_RX; i < MLX5DR_TABLE_TYPE_MAX; i++) {
if (priv->hw_pop_vlan[i]) {
mlx5dr_action_destroy(priv->hw_pop_vlan[i]);
priv->hw_pop_vlan[i] = NULL;
}
if (priv->hw_push_vlan[i]) {
mlx5dr_action_destroy(priv->hw_push_vlan[i]);
priv->hw_push_vlan[i] = NULL;
}
}
}
static int
flow_hw_create_vlan(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
enum mlx5dr_table_type i;
const enum mlx5dr_action_flags flags[MLX5DR_TABLE_TYPE_MAX] = {
MLX5DR_ACTION_FLAG_HWS_RX,
MLX5DR_ACTION_FLAG_HWS_TX,
MLX5DR_ACTION_FLAG_HWS_FDB
};
for (i = MLX5DR_TABLE_TYPE_NIC_RX; i <= MLX5DR_TABLE_TYPE_NIC_TX; i++) {
priv->hw_pop_vlan[i] =
mlx5dr_action_create_pop_vlan(priv->dr_ctx, flags[i]);
if (!priv->hw_pop_vlan[i])
return -ENOENT;
priv->hw_push_vlan[i] =
mlx5dr_action_create_push_vlan(priv->dr_ctx, flags[i]);
if (!priv->hw_pop_vlan[i])
return -ENOENT;
}
if (priv->sh->config.dv_esw_en && priv->master) {
priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB] =
mlx5dr_action_create_pop_vlan
(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
return -ENOENT;
priv->hw_push_vlan[MLX5DR_TABLE_TYPE_FDB] =
mlx5dr_action_create_push_vlan
(priv->dr_ctx, MLX5DR_ACTION_FLAG_HWS_FDB);
if (!priv->hw_pop_vlan[MLX5DR_TABLE_TYPE_FDB])
return -ENOENT;
}
return 0;
}
static void
flow_hw_cleanup_ctrl_rx_tables(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
unsigned int i;
unsigned int j;
if (!priv->hw_ctrl_rx)
return;
for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
struct rte_flow_template_table *tbl = priv->hw_ctrl_rx->tables[i][j].tbl;
struct rte_flow_pattern_template *pt = priv->hw_ctrl_rx->tables[i][j].pt;
if (tbl)
claim_zero(flow_hw_table_destroy(dev, tbl, NULL));
if (pt)
claim_zero(flow_hw_pattern_template_destroy(dev, pt, NULL));
}
}
for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++i) {
struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[i];
if (at)
claim_zero(flow_hw_actions_template_destroy(dev, at, NULL));
}
mlx5_free(priv->hw_ctrl_rx);
priv->hw_ctrl_rx = NULL;
}
static uint64_t
flow_hw_ctrl_rx_rss_type_hash_types(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
switch (rss_type) {
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP:
return 0;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
return RTE_ETH_RSS_IPV4 | RTE_ETH_RSS_FRAG_IPV4 | RTE_ETH_RSS_NONFRAG_IPV4_OTHER;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
return RTE_ETH_RSS_NONFRAG_IPV4_UDP;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
return RTE_ETH_RSS_NONFRAG_IPV4_TCP;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
return RTE_ETH_RSS_IPV6 | RTE_ETH_RSS_FRAG_IPV6 | RTE_ETH_RSS_NONFRAG_IPV6_OTHER;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
return RTE_ETH_RSS_NONFRAG_IPV6_UDP | RTE_ETH_RSS_IPV6_UDP_EX;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
return RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_IPV6_TCP_EX;
default:
/* Should not reach here. */
MLX5_ASSERT(false);
return 0;
}
}
static struct rte_flow_actions_template *
flow_hw_create_ctrl_rx_rss_template(struct rte_eth_dev *dev,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_actions_template_attr attr = {
.ingress = 1,
};
uint16_t queue[RTE_MAX_QUEUES_PER_PORT];
struct rte_flow_action_rss rss_conf = {
.func = RTE_ETH_HASH_FUNCTION_DEFAULT,
.level = 0,
.types = 0,
.key_len = priv->rss_conf.rss_key_len,
.key = priv->rss_conf.rss_key,
.queue_num = priv->reta_idx_n,
.queue = queue,
};
struct rte_flow_action actions[] = {
{
.type = RTE_FLOW_ACTION_TYPE_RSS,
.conf = &rss_conf,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_flow_action masks[] = {
{
.type = RTE_FLOW_ACTION_TYPE_RSS,
.conf = &rss_conf,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_flow_actions_template *at;
struct rte_flow_error error;
unsigned int i;
MLX5_ASSERT(priv->reta_idx_n > 0 && priv->reta_idx);
/* Select proper RSS hash types and based on that configure the actions template. */
rss_conf.types = flow_hw_ctrl_rx_rss_type_hash_types(rss_type);
if (rss_conf.types) {
for (i = 0; i < priv->reta_idx_n; ++i)
queue[i] = (*priv->reta_idx)[i];
} else {
rss_conf.queue_num = 1;
queue[0] = (*priv->reta_idx)[0];
}
at = flow_hw_actions_template_create(dev, &attr, actions, masks, &error);
if (!at)
DRV_LOG(ERR,
"Failed to create ctrl flow actions template: rte_errno(%d), type(%d): %s",
rte_errno, error.type,
error.message ? error.message : "(no stated reason)");
return at;
}
static uint32_t ctrl_rx_rss_priority_map[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX] = {
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_NON_IP] = MLX5_HW_CTRL_RX_PRIO_L2,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4] = MLX5_HW_CTRL_RX_PRIO_L3,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6] = MLX5_HW_CTRL_RX_PRIO_L3,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP] = MLX5_HW_CTRL_RX_PRIO_L4,
[MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP] = MLX5_HW_CTRL_RX_PRIO_L4,
};
static uint32_t ctrl_rx_nb_flows_map[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX] = {
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL] = 1,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST] = 1,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST] = 1,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN] = MLX5_MAX_VLAN_IDS,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST] = 1,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST] = 1,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN] = MLX5_MAX_VLAN_IDS,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC] = MLX5_MAX_UC_MAC_ADDRESSES,
[MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN] =
MLX5_MAX_UC_MAC_ADDRESSES * MLX5_MAX_VLAN_IDS,
};
static struct rte_flow_template_table_attr
flow_hw_get_ctrl_rx_table_attr(enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
return (struct rte_flow_template_table_attr){
.flow_attr = {
.group = 0,
.priority = ctrl_rx_rss_priority_map[rss_type],
.ingress = 1,
},
.nb_flows = ctrl_rx_nb_flows_map[eth_pattern_type],
};
}
static struct rte_flow_item
flow_hw_get_ctrl_rx_eth_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
struct rte_flow_item item = {
.type = RTE_FLOW_ITEM_TYPE_ETH,
.mask = NULL,
};
switch (eth_pattern_type) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
item.mask = &ctrl_rx_eth_promisc_mask;
break;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
item.mask = &ctrl_rx_eth_mcast_mask;
break;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
item.mask = &ctrl_rx_eth_dmac_mask;
break;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
item.mask = &ctrl_rx_eth_ipv4_mcast_mask;
break;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
item.mask = &ctrl_rx_eth_ipv6_mcast_mask;
break;
default:
/* Should not reach here - ETH mask must be present. */
item.type = RTE_FLOW_ITEM_TYPE_END;
MLX5_ASSERT(false);
break;
}
return item;
}
static struct rte_flow_item
flow_hw_get_ctrl_rx_vlan_item(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
struct rte_flow_item item = {
.type = RTE_FLOW_ITEM_TYPE_VOID,
.mask = NULL,
};
switch (eth_pattern_type) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
item.type = RTE_FLOW_ITEM_TYPE_VLAN;
item.mask = &rte_flow_item_vlan_mask;
break;
default:
/* Nothing to update. */
break;
}
return item;
}
static struct rte_flow_item
flow_hw_get_ctrl_rx_l3_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct rte_flow_item item = {
.type = RTE_FLOW_ITEM_TYPE_VOID,
.mask = NULL,
};
switch (rss_type) {
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
item.type = RTE_FLOW_ITEM_TYPE_IPV4;
break;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
item.type = RTE_FLOW_ITEM_TYPE_IPV6;
break;
default:
/* Nothing to update. */
break;
}
return item;
}
static struct rte_flow_item
flow_hw_get_ctrl_rx_l4_item(const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct rte_flow_item item = {
.type = RTE_FLOW_ITEM_TYPE_VOID,
.mask = NULL,
};
switch (rss_type) {
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_UDP:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_UDP:
item.type = RTE_FLOW_ITEM_TYPE_UDP;
break;
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV4_TCP:
case MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_IPV6_TCP:
item.type = RTE_FLOW_ITEM_TYPE_TCP;
break;
default:
/* Nothing to update. */
break;
}
return item;
}
static struct rte_flow_pattern_template *
flow_hw_create_ctrl_rx_pattern_template
(struct rte_eth_dev *dev,
const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
const struct rte_flow_pattern_template_attr attr = {
.relaxed_matching = 0,
.ingress = 1,
};
struct rte_flow_item items[] = {
/* Matching patterns */
flow_hw_get_ctrl_rx_eth_item(eth_pattern_type),
flow_hw_get_ctrl_rx_vlan_item(eth_pattern_type),
flow_hw_get_ctrl_rx_l3_item(rss_type),
flow_hw_get_ctrl_rx_l4_item(rss_type),
/* Terminate pattern */
{ .type = RTE_FLOW_ITEM_TYPE_END }
};
return flow_hw_pattern_template_create(dev, &attr, items, NULL);
}
static int
flow_hw_create_ctrl_rx_tables(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
unsigned int i;
unsigned int j;
int ret;
MLX5_ASSERT(!priv->hw_ctrl_rx);
priv->hw_ctrl_rx = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*priv->hw_ctrl_rx),
RTE_CACHE_LINE_SIZE, rte_socket_id());
if (!priv->hw_ctrl_rx) {
DRV_LOG(ERR, "Failed to allocate memory for Rx control flow tables");
rte_errno = ENOMEM;
return -rte_errno;
}
/* Create all pattern template variants. */
for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;
for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
struct rte_flow_template_table_attr attr;
struct rte_flow_pattern_template *pt;
attr = flow_hw_get_ctrl_rx_table_attr(eth_pattern_type, rss_type);
pt = flow_hw_create_ctrl_rx_pattern_template(dev, eth_pattern_type,
rss_type);
if (!pt)
goto err;
priv->hw_ctrl_rx->tables[i][j].attr = attr;
priv->hw_ctrl_rx->tables[i][j].pt = pt;
}
}
return 0;
err:
ret = rte_errno;
flow_hw_cleanup_ctrl_rx_tables(dev);
rte_errno = ret;
return -ret;
}
void
mlx5_flow_hw_cleanup_ctrl_rx_templates(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
unsigned int i;
unsigned int j;
if (!priv->dr_ctx)
return;
if (!priv->hw_ctrl_rx)
return;
hw_ctrl_rx = priv->hw_ctrl_rx;
for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];
if (tmpls->tbl) {
claim_zero(flow_hw_table_destroy(dev, tmpls->tbl, NULL));
tmpls->tbl = NULL;
}
}
}
for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
if (hw_ctrl_rx->rss[j]) {
claim_zero(flow_hw_actions_template_destroy(dev, hw_ctrl_rx->rss[j], NULL));
hw_ctrl_rx->rss[j] = NULL;
}
}
}
/**
* Configure port HWS resources.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] port_attr
* Port configuration attributes.
* @param[in] nb_queue
* Number of queue.
* @param[in] queue_attr
* Array that holds attributes for each flow queue.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_configure(struct rte_eth_dev *dev,
const struct rte_flow_port_attr *port_attr,
uint16_t nb_queue,
const struct rte_flow_queue_attr *queue_attr[],
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5dr_context *dr_ctx = NULL;
struct mlx5dr_context_attr dr_ctx_attr = {0};
struct mlx5_hw_q *hw_q;
struct mlx5_hw_q_job *job = NULL;
uint32_t mem_size, i, j;
struct mlx5_indexed_pool_config cfg = {
.size = sizeof(struct mlx5_action_construct_data),
.trunk_size = 4096,
.need_lock = 1,
.release_mem_en = !!priv->sh->config.reclaim_mode,
.malloc = mlx5_malloc,
.free = mlx5_free,
.type = "mlx5_hw_action_construct_data",
};
/* Adds one queue to be used by PMD.
* The last queue will be used by the PMD.
*/
uint16_t nb_q_updated = 0;
struct rte_flow_queue_attr **_queue_attr = NULL;
struct rte_flow_queue_attr ctrl_queue_attr = {0};
bool is_proxy = !!(priv->sh->config.dv_esw_en && priv->master);
int ret = 0;
if (!port_attr || !nb_queue || !queue_attr) {
rte_errno = EINVAL;
goto err;
}
/* In case re-configuring, release existing context at first. */
if (priv->dr_ctx) {
/* */
for (i = 0; i < priv->nb_queue; i++) {
hw_q = &priv->hw_q[i];
/* Make sure all queues are empty. */
if (hw_q->size != hw_q->job_idx) {
rte_errno = EBUSY;
goto err;
}
}
flow_hw_resource_release(dev);
}
ctrl_queue_attr.size = queue_attr[0]->size;
nb_q_updated = nb_queue + 1;
_queue_attr = mlx5_malloc(MLX5_MEM_ZERO,
nb_q_updated *
sizeof(struct rte_flow_queue_attr *),
64, SOCKET_ID_ANY);
if (!_queue_attr) {
rte_errno = ENOMEM;
goto err;
}
memcpy(_queue_attr, queue_attr,
sizeof(void *) * nb_queue);
_queue_attr[nb_queue] = &ctrl_queue_attr;
priv->acts_ipool = mlx5_ipool_create(&cfg);
if (!priv->acts_ipool)
goto err;
/* Allocate the queue job descriptor LIFO. */
mem_size = sizeof(priv->hw_q[0]) * nb_q_updated;
for (i = 0; i < nb_q_updated; i++) {
/*
* Check if the queues' size are all the same as the
* limitation from HWS layer.
*/
if (_queue_attr[i]->size != _queue_attr[0]->size) {
rte_errno = EINVAL;
goto err;
}
mem_size += (sizeof(struct mlx5_hw_q_job *) +
sizeof(struct mlx5_hw_q_job) +
sizeof(uint8_t) * MLX5_ENCAP_MAX_LEN +
sizeof(struct mlx5_modification_cmd) *
MLX5_MHDR_MAX_CMD +
sizeof(struct rte_flow_item) *
MLX5_HW_MAX_ITEMS) *
_queue_attr[i]->size;
}
priv->hw_q = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
64, SOCKET_ID_ANY);
if (!priv->hw_q) {
rte_errno = ENOMEM;
goto err;
}
for (i = 0; i < nb_q_updated; i++) {
char mz_name[RTE_MEMZONE_NAMESIZE];
uint8_t *encap = NULL;
struct mlx5_modification_cmd *mhdr_cmd = NULL;
struct rte_flow_item *items = NULL;
priv->hw_q[i].job_idx = _queue_attr[i]->size;
priv->hw_q[i].size = _queue_attr[i]->size;
if (i == 0)
priv->hw_q[i].job = (struct mlx5_hw_q_job **)
&priv->hw_q[nb_q_updated];
else
priv->hw_q[i].job = (struct mlx5_hw_q_job **)
&job[_queue_attr[i - 1]->size - 1].items
[MLX5_HW_MAX_ITEMS];
job = (struct mlx5_hw_q_job *)
&priv->hw_q[i].job[_queue_attr[i]->size];
mhdr_cmd = (struct mlx5_modification_cmd *)
&job[_queue_attr[i]->size];
encap = (uint8_t *)
&mhdr_cmd[_queue_attr[i]->size * MLX5_MHDR_MAX_CMD];
items = (struct rte_flow_item *)
&encap[_queue_attr[i]->size * MLX5_ENCAP_MAX_LEN];
for (j = 0; j < _queue_attr[i]->size; j++) {
job[j].mhdr_cmd = &mhdr_cmd[j * MLX5_MHDR_MAX_CMD];
job[j].encap_data = &encap[j * MLX5_ENCAP_MAX_LEN];
job[j].items = &items[j * MLX5_HW_MAX_ITEMS];
priv->hw_q[i].job[j] = &job[j];
}
snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_cq_%u",
dev->data->port_id, i);
priv->hw_q[i].indir_cq = rte_ring_create(mz_name,
_queue_attr[i]->size, SOCKET_ID_ANY,
RING_F_SP_ENQ | RING_F_SC_DEQ |
RING_F_EXACT_SZ);
if (!priv->hw_q[i].indir_cq)
goto err;
snprintf(mz_name, sizeof(mz_name), "port_%u_indir_act_iq_%u",
dev->data->port_id, i);
priv->hw_q[i].indir_iq = rte_ring_create(mz_name,
_queue_attr[i]->size, SOCKET_ID_ANY,
RING_F_SP_ENQ | RING_F_SC_DEQ |
RING_F_EXACT_SZ);
if (!priv->hw_q[i].indir_iq)
goto err;
}
dr_ctx_attr.pd = priv->sh->cdev->pd;
dr_ctx_attr.queues = nb_q_updated;
/* Queue size should all be the same. Take the first one. */
dr_ctx_attr.queue_size = _queue_attr[0]->size;
dr_ctx = mlx5dr_context_open(priv->sh->cdev->ctx, &dr_ctx_attr);
/* rte_errno has been updated by HWS layer. */
if (!dr_ctx)
goto err;
priv->dr_ctx = dr_ctx;
priv->nb_queue = nb_q_updated;
rte_spinlock_init(&priv->hw_ctrl_lock);
LIST_INIT(&priv->hw_ctrl_flows);
ret = flow_hw_create_ctrl_rx_tables(dev);
if (ret) {
rte_flow_error_set(error, -ret, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Failed to set up Rx control flow templates");
goto err;
}
/* Initialize meter library*/
if (port_attr->nb_meters)
if (mlx5_flow_meter_init(dev, port_attr->nb_meters, 1, 1, nb_q_updated))
goto err;
/* Add global actions. */
for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
uint32_t act_flags = 0;
act_flags = mlx5_hw_act_flag[i][0] | mlx5_hw_act_flag[i][1];
if (is_proxy)
act_flags |= mlx5_hw_act_flag[i][2];
priv->hw_drop[i] = mlx5dr_action_create_dest_drop(priv->dr_ctx, act_flags);
if (!priv->hw_drop[i])
goto err;
priv->hw_tag[i] = mlx5dr_action_create_tag
(priv->dr_ctx, mlx5_hw_act_flag[i][0]);
if (!priv->hw_tag[i])
goto err;
}
if (priv->sh->config.dv_esw_en && priv->sh->config.repr_matching) {
ret = flow_hw_setup_tx_repr_tagging(dev);
if (ret) {
rte_errno = -ret;
goto err;
}
}
if (is_proxy) {
ret = flow_hw_create_vport_actions(priv);
if (ret) {
rte_errno = -ret;
goto err;
}
ret = flow_hw_create_ctrl_tables(dev);
if (ret) {
rte_errno = -ret;
goto err;
}
}
if (port_attr->nb_conn_tracks) {
mem_size = sizeof(struct mlx5_aso_sq) * nb_q_updated +
sizeof(*priv->ct_mng);
priv->ct_mng = mlx5_malloc(MLX5_MEM_ZERO, mem_size,
RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (!priv->ct_mng)
goto err;
if (mlx5_aso_ct_queue_init(priv->sh, priv->ct_mng, nb_q_updated))
goto err;
priv->hws_ctpool = flow_hw_ct_pool_create(dev, port_attr);
if (!priv->hws_ctpool)
goto err;
priv->sh->ct_aso_en = 1;
}
if (port_attr->nb_counters) {
priv->hws_cpool = mlx5_hws_cnt_pool_create(dev, port_attr,
nb_queue);
if (priv->hws_cpool == NULL)
goto err;
}
if (port_attr->nb_aging_objects) {
if (port_attr->nb_counters == 0) {
/*
* Aging management uses counter. Number counters
* requesting should take into account a counter for
* each flow rules containing AGE without counter.
*/
DRV_LOG(ERR, "Port %u AGE objects are requested (%u) "
"without counters requesting.",
dev->data->port_id,
port_attr->nb_aging_objects);
rte_errno = EINVAL;
goto err;
}
ret = mlx5_hws_age_pool_init(dev, port_attr, nb_queue);
if (ret < 0)
goto err;
}
ret = flow_hw_create_vlan(dev);
if (ret)
goto err;
if (_queue_attr)
mlx5_free(_queue_attr);
if (port_attr->flags & RTE_FLOW_PORT_FLAG_STRICT_QUEUE)
priv->hws_strict_queue = 1;
return 0;
err:
if (priv->hws_ctpool) {
flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
flow_hw_ct_mng_destroy(dev, priv->ct_mng);
priv->ct_mng = NULL;
}
if (priv->hws_age_req)
mlx5_hws_age_pool_destroy(priv);
if (priv->hws_cpool) {
mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
priv->hws_cpool = NULL;
}
flow_hw_free_vport_actions(priv);
for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
if (priv->hw_drop[i])
mlx5dr_action_destroy(priv->hw_drop[i]);
if (priv->hw_tag[i])
mlx5dr_action_destroy(priv->hw_tag[i]);
}
flow_hw_destroy_vlan(dev);
if (dr_ctx)
claim_zero(mlx5dr_context_close(dr_ctx));
for (i = 0; i < nb_q_updated; i++) {
if (priv->hw_q[i].indir_iq)
rte_ring_free(priv->hw_q[i].indir_iq);
if (priv->hw_q[i].indir_cq)
rte_ring_free(priv->hw_q[i].indir_cq);
}
mlx5_free(priv->hw_q);
priv->hw_q = NULL;
if (priv->acts_ipool) {
mlx5_ipool_destroy(priv->acts_ipool);
priv->acts_ipool = NULL;
}
if (_queue_attr)
mlx5_free(_queue_attr);
/* Do not overwrite the internal errno information. */
if (ret)
return ret;
return rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"fail to configure port");
}
/**
* Release HWS resources.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
*/
void
flow_hw_resource_release(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_template_table *tbl;
struct rte_flow_pattern_template *it;
struct rte_flow_actions_template *at;
uint32_t i;
if (!priv->dr_ctx)
return;
flow_hw_rxq_flag_set(dev, false);
flow_hw_flush_all_ctrl_flows(dev);
flow_hw_cleanup_tx_repr_tagging(dev);
flow_hw_cleanup_ctrl_rx_tables(dev);
while (!LIST_EMPTY(&priv->flow_hw_tbl_ongo)) {
tbl = LIST_FIRST(&priv->flow_hw_tbl_ongo);
flow_hw_table_destroy(dev, tbl, NULL);
}
while (!LIST_EMPTY(&priv->flow_hw_tbl)) {
tbl = LIST_FIRST(&priv->flow_hw_tbl);
flow_hw_table_destroy(dev, tbl, NULL);
}
while (!LIST_EMPTY(&priv->flow_hw_itt)) {
it = LIST_FIRST(&priv->flow_hw_itt);
flow_hw_pattern_template_destroy(dev, it, NULL);
}
while (!LIST_EMPTY(&priv->flow_hw_at)) {
at = LIST_FIRST(&priv->flow_hw_at);
flow_hw_actions_template_destroy(dev, at, NULL);
}
for (i = 0; i < MLX5_HW_ACTION_FLAG_MAX; i++) {
if (priv->hw_drop[i])
mlx5dr_action_destroy(priv->hw_drop[i]);
if (priv->hw_tag[i])
mlx5dr_action_destroy(priv->hw_tag[i]);
}
flow_hw_destroy_vlan(dev);
flow_hw_free_vport_actions(priv);
if (priv->acts_ipool) {
mlx5_ipool_destroy(priv->acts_ipool);
priv->acts_ipool = NULL;
}
if (priv->hws_age_req)
mlx5_hws_age_pool_destroy(priv);
if (priv->hws_cpool) {
mlx5_hws_cnt_pool_destroy(priv->sh, priv->hws_cpool);
priv->hws_cpool = NULL;
}
if (priv->hws_ctpool) {
flow_hw_ct_pool_destroy(dev, priv->hws_ctpool);
priv->hws_ctpool = NULL;
}
if (priv->ct_mng) {
flow_hw_ct_mng_destroy(dev, priv->ct_mng);
priv->ct_mng = NULL;
}
for (i = 0; i < priv->nb_queue; i++) {
rte_ring_free(priv->hw_q[i].indir_iq);
rte_ring_free(priv->hw_q[i].indir_cq);
}
mlx5_free(priv->hw_q);
priv->hw_q = NULL;
claim_zero(mlx5dr_context_close(priv->dr_ctx));
priv->dr_ctx = NULL;
priv->nb_queue = 0;
}
/* Sets vport tag and mask, for given port, used in HWS rules. */
void
flow_hw_set_port_info(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint16_t port_id = dev->data->port_id;
struct flow_hw_port_info *info;
MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
info = &mlx5_flow_hw_port_infos[port_id];
info->regc_mask = priv->vport_meta_mask;
info->regc_value = priv->vport_meta_tag;
info->is_wire = priv->master;
}
/* Clears vport tag and mask used for HWS rules. */
void
flow_hw_clear_port_info(struct rte_eth_dev *dev)
{
uint16_t port_id = dev->data->port_id;
struct flow_hw_port_info *info;
MLX5_ASSERT(port_id < RTE_MAX_ETHPORTS);
info = &mlx5_flow_hw_port_infos[port_id];
info->regc_mask = 0;
info->regc_value = 0;
info->is_wire = 0;
}
/*
* Initialize the information of available tag registers and an intersection
* of all the probed devices' REG_C_Xs.
* PS. No port concept in steering part, right now it cannot be per port level.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
*/
void flow_hw_init_tags_set(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t meta_mode = priv->sh->config.dv_xmeta_en;
uint8_t masks = (uint8_t)priv->sh->cdev->config.hca_attr.set_reg_c;
uint32_t i, j;
enum modify_reg copy[MLX5_FLOW_HW_TAGS_MAX] = {REG_NON};
uint8_t unset = 0;
uint8_t copy_masks = 0;
/*
* The CAPA is global for common device but only used in net.
* It is shared per eswitch domain.
*/
if (!!priv->sh->hws_tags)
return;
unset |= 1 << (priv->mtr_color_reg - REG_C_0);
unset |= 1 << (REG_C_6 - REG_C_0);
if (priv->sh->config.dv_esw_en)
unset |= 1 << (REG_C_0 - REG_C_0);
if (meta_mode == MLX5_XMETA_MODE_META32_HWS)
unset |= 1 << (REG_C_1 - REG_C_0);
masks &= ~unset;
if (mlx5_flow_hw_avl_tags_init_cnt) {
MLX5_ASSERT(mlx5_flow_hw_aso_tag == priv->mtr_color_reg);
for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
if (mlx5_flow_hw_avl_tags[i] != REG_NON && !!((1 << i) & masks)) {
copy[mlx5_flow_hw_avl_tags[i] - REG_C_0] =
mlx5_flow_hw_avl_tags[i];
copy_masks |= (1 << (mlx5_flow_hw_avl_tags[i] - REG_C_0));
}
}
if (copy_masks != masks) {
j = 0;
for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++)
if (!!((1 << i) & copy_masks))
mlx5_flow_hw_avl_tags[j++] = copy[i];
}
} else {
j = 0;
for (i = 0; i < MLX5_FLOW_HW_TAGS_MAX; i++) {
if (!!((1 << i) & masks))
mlx5_flow_hw_avl_tags[j++] =
(enum modify_reg)(i + (uint32_t)REG_C_0);
}
}
priv->sh->hws_tags = 1;
mlx5_flow_hw_aso_tag = (enum modify_reg)priv->mtr_color_reg;
mlx5_flow_hw_avl_tags_init_cnt++;
}
/*
* Reset the available tag registers information to NONE.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
*/
void flow_hw_clear_tags_set(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
if (!priv->sh->hws_tags)
return;
priv->sh->hws_tags = 0;
mlx5_flow_hw_avl_tags_init_cnt--;
if (!mlx5_flow_hw_avl_tags_init_cnt)
memset(mlx5_flow_hw_avl_tags, REG_NON,
sizeof(enum modify_reg) * MLX5_FLOW_HW_TAGS_MAX);
}
uint32_t mlx5_flow_hw_flow_metadata_config_refcnt;
uint8_t mlx5_flow_hw_flow_metadata_esw_en;
uint8_t mlx5_flow_hw_flow_metadata_xmeta_en;
/**
* Initializes static configuration of META flow items.
*
* As a temporary workaround, META flow item is translated to a register,
* based on statically saved dv_esw_en and dv_xmeta_en device arguments.
* It is a workaround for flow_hw_get_reg_id() where port specific information
* is not available at runtime.
*
* Values of dv_esw_en and dv_xmeta_en device arguments are taken from the first opened port.
* This means that each mlx5 port will use the same configuration for translation
* of META flow items.
*
* @param[in] dev
* Pointer to Ethernet device.
*/
void
flow_hw_init_flow_metadata_config(struct rte_eth_dev *dev)
{
uint32_t refcnt;
refcnt = __atomic_fetch_add(&mlx5_flow_hw_flow_metadata_config_refcnt, 1,
__ATOMIC_RELAXED);
if (refcnt > 0)
return;
mlx5_flow_hw_flow_metadata_esw_en = MLX5_SH(dev)->config.dv_esw_en;
mlx5_flow_hw_flow_metadata_xmeta_en = MLX5_SH(dev)->config.dv_xmeta_en;
}
/**
* Clears statically stored configuration related to META flow items.
*/
void
flow_hw_clear_flow_metadata_config(void)
{
uint32_t refcnt;
refcnt = __atomic_sub_fetch(&mlx5_flow_hw_flow_metadata_config_refcnt, 1,
__ATOMIC_RELAXED);
if (refcnt > 0)
return;
mlx5_flow_hw_flow_metadata_esw_en = 0;
mlx5_flow_hw_flow_metadata_xmeta_en = 0;
}
static int
flow_hw_conntrack_destroy(struct rte_eth_dev *dev __rte_unused,
uint32_t idx,
struct rte_flow_error *error)
{
uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
uint32_t ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
struct rte_eth_dev *owndev = &rte_eth_devices[owner];
struct mlx5_priv *priv = owndev->data->dev_private;
struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
struct mlx5_aso_ct_action *ct;
ct = mlx5_ipool_get(pool->cts, ct_idx);
if (!ct) {
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Invalid CT destruction index");
}
__atomic_store_n(&ct->state, ASO_CONNTRACK_FREE,
__ATOMIC_RELAXED);
mlx5_ipool_free(pool->cts, ct_idx);
return 0;
}
static int
flow_hw_conntrack_query(struct rte_eth_dev *dev, uint32_t queue, uint32_t idx,
struct rte_flow_action_conntrack *profile,
void *user_data, bool push,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
struct mlx5_aso_ct_action *ct;
uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
uint32_t ct_idx;
if (owner != PORT_ID(priv))
return rte_flow_error_set(error, EACCES,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Can't query CT object owned by another port");
ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
ct = mlx5_ipool_get(pool->cts, ct_idx);
if (!ct) {
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Invalid CT query index");
}
profile->peer_port = ct->peer;
profile->is_original_dir = ct->is_original;
if (mlx5_aso_ct_query_by_wqe(priv->sh, queue, ct, profile, user_data, push))
return rte_flow_error_set(error, EIO,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Failed to query CT context");
return 0;
}
static int
flow_hw_conntrack_update(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_modify_conntrack *action_conf,
uint32_t idx, void *user_data, bool push,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
struct mlx5_aso_ct_action *ct;
const struct rte_flow_action_conntrack *new_prf;
uint16_t owner = (uint16_t)MLX5_ACTION_CTX_CT_GET_OWNER(idx);
uint32_t ct_idx;
int ret = 0;
if (PORT_ID(priv) != owner)
return rte_flow_error_set(error, EACCES,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Can't update CT object owned by another port");
ct_idx = MLX5_ACTION_CTX_CT_GET_IDX(idx);
ct = mlx5_ipool_get(pool->cts, ct_idx);
if (!ct) {
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Invalid CT update index");
}
new_prf = &action_conf->new_ct;
if (action_conf->direction)
ct->is_original = !!new_prf->is_original_dir;
if (action_conf->state) {
/* Only validate the profile when it needs to be updated. */
ret = mlx5_validate_action_ct(dev, new_prf, error);
if (ret)
return ret;
ret = mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, new_prf,
user_data, push);
if (ret)
return rte_flow_error_set(error, EIO,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Failed to send CT context update WQE");
if (queue != MLX5_HW_INV_QUEUE)
return 0;
/* Block until ready or a failure in synchronous mode. */
ret = mlx5_aso_ct_available(priv->sh, queue, ct);
if (ret)
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Timeout to get the CT update");
}
return ret;
}
static struct rte_flow_action_handle *
flow_hw_conntrack_create(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_action_conntrack *pro,
void *user_data, bool push,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_ct_pool *pool = priv->hws_ctpool;
struct mlx5_aso_ct_action *ct;
uint32_t ct_idx = 0;
int ret;
bool async = !!(queue != MLX5_HW_INV_QUEUE);
if (!pool) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"CT is not enabled");
return 0;
}
ct = mlx5_ipool_zmalloc(pool->cts, &ct_idx);
if (!ct) {
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"Failed to allocate CT object");
return 0;
}
ct->offset = ct_idx - 1;
ct->is_original = !!pro->is_original_dir;
ct->peer = pro->peer_port;
ct->pool = pool;
if (mlx5_aso_ct_update_by_wqe(priv->sh, queue, ct, pro, user_data, push)) {
mlx5_ipool_free(pool->cts, ct_idx);
rte_flow_error_set(error, EBUSY,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"Failed to update CT");
return 0;
}
if (!async) {
ret = mlx5_aso_ct_available(priv->sh, queue, ct);
if (ret) {
mlx5_ipool_free(pool->cts, ct_idx);
rte_flow_error_set(error, rte_errno,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL,
"Timeout to get the CT update");
return 0;
}
}
return (struct rte_flow_action_handle *)(uintptr_t)
MLX5_ACTION_CTX_CT_GEN_IDX(PORT_ID(priv), ct_idx);
}
/**
* Validate shared action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* Which queue to be used.
* @param[in] attr
* Operation attribute.
* @param[in] conf
* Indirect action configuration.
* @param[in] action
* rte_flow action detail.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, otherwise negative errno value.
*/
static int
flow_hw_action_handle_validate(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_op_attr *attr,
const struct rte_flow_indir_action_conf *conf,
const struct rte_flow_action *action,
void *user_data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
RTE_SET_USED(attr);
RTE_SET_USED(queue);
RTE_SET_USED(user_data);
switch (action->type) {
case RTE_FLOW_ACTION_TYPE_AGE:
if (!priv->hws_age_req)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"aging pool not initialized");
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
if (!priv->hws_cpool)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"counters pool not initialized");
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
if (priv->hws_ctpool == NULL)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"CT pool not initialized");
return mlx5_validate_action_ct(dev, action->conf, error);
case RTE_FLOW_ACTION_TYPE_METER_MARK:
return flow_hw_validate_action_meter_mark(dev, action, error);
case RTE_FLOW_ACTION_TYPE_RSS:
return flow_dv_action_validate(dev, conf, action, error);
default:
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"action type not supported");
}
return 0;
}
/**
* Create shared action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* Which queue to be used.
* @param[in] attr
* Operation attribute.
* @param[in] conf
* Indirect action configuration.
* @param[in] action
* rte_flow action detail.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* Action handle on success, NULL otherwise and rte_errno is set.
*/
static struct rte_flow_action_handle *
flow_hw_action_handle_create(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_op_attr *attr,
const struct rte_flow_indir_action_conf *conf,
const struct rte_flow_action *action,
void *user_data,
struct rte_flow_error *error)
{
struct rte_flow_action_handle *handle = NULL;
struct mlx5_hw_q_job *job = NULL;
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_action_age *age;
struct mlx5_aso_mtr *aso_mtr;
cnt_id_t cnt_id;
uint32_t mtr_id;
uint32_t age_idx;
bool push = true;
bool aso = false;
if (attr) {
MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
if (unlikely(!priv->hw_q[queue].job_idx)) {
rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Flow queue full.");
return NULL;
}
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
job->type = MLX5_HW_Q_JOB_TYPE_CREATE;
job->user_data = user_data;
push = !attr->postpone;
}
switch (action->type) {
case RTE_FLOW_ACTION_TYPE_AGE:
if (priv->hws_strict_queue) {
struct mlx5_age_info *info = GET_PORT_AGE_INFO(priv);
if (queue >= info->hw_q_age->nb_rings) {
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"Invalid queue ID for indirect AGE.");
rte_errno = EINVAL;
return NULL;
}
}
age = action->conf;
age_idx = mlx5_hws_age_action_create(priv, queue, true, age,
0, error);
if (age_idx == 0) {
rte_flow_error_set(error, ENODEV,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"AGE are not configured!");
} else {
age_idx = (MLX5_INDIRECT_ACTION_TYPE_AGE <<
MLX5_INDIRECT_ACTION_TYPE_OFFSET) | age_idx;
handle =
(struct rte_flow_action_handle *)(uintptr_t)age_idx;
}
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
if (mlx5_hws_cnt_shared_get(priv->hws_cpool, &cnt_id, 0))
rte_flow_error_set(error, ENODEV,
RTE_FLOW_ERROR_TYPE_ACTION,
NULL,
"counter are not configured!");
else
handle = (struct rte_flow_action_handle *)
(uintptr_t)cnt_id;
break;
case RTE_FLOW_ACTION_TYPE_CONNTRACK:
aso = true;
handle = flow_hw_conntrack_create(dev, queue, action->conf, job,
push, error);
break;
case RTE_FLOW_ACTION_TYPE_METER_MARK:
aso = true;
aso_mtr = flow_hw_meter_mark_alloc(dev, queue, action, job, push);
if (!aso_mtr)
break;
mtr_id = (MLX5_INDIRECT_ACTION_TYPE_METER_MARK <<
MLX5_INDIRECT_ACTION_TYPE_OFFSET) | (aso_mtr->fm.meter_id);
handle = (struct rte_flow_action_handle *)(uintptr_t)mtr_id;
break;
case RTE_FLOW_ACTION_TYPE_RSS:
handle = flow_dv_action_create(dev, conf, action, error);
break;
default:
rte_flow_error_set(error, ENOTSUP, RTE_FLOW_ERROR_TYPE_ACTION,
NULL, "action type not supported");
break;
}
if (job) {
if (!handle) {
priv->hw_q[queue].job_idx++;
return NULL;
}
job->action = handle;
if (push)
__flow_hw_push_action(dev, queue);
if (aso)
return handle;
rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
priv->hw_q[queue].indir_iq, job);
}
return handle;
}
/**
* Update shared action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* Which queue to be used.
* @param[in] attr
* Operation attribute.
* @param[in] handle
* Action handle to be updated.
* @param[in] update
* Update value.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_action_handle_update(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_op_attr *attr,
struct rte_flow_action_handle *handle,
const void *update,
void *user_data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
const struct rte_flow_modify_conntrack *ct_conf =
(const struct rte_flow_modify_conntrack *)update;
const struct rte_flow_update_meter_mark *upd_meter_mark =
(const struct rte_flow_update_meter_mark *)update;
const struct rte_flow_action_meter_mark *meter_mark;
struct mlx5_hw_q_job *job = NULL;
struct mlx5_aso_mtr *aso_mtr;
struct mlx5_flow_meter_info *fm;
uint32_t act_idx = (uint32_t)(uintptr_t)handle;
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
int ret = 0;
bool push = true;
bool aso = false;
if (attr) {
MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
if (unlikely(!priv->hw_q[queue].job_idx))
return rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Action update failed due to queue full.");
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
job->type = MLX5_HW_Q_JOB_TYPE_UPDATE;
job->user_data = user_data;
push = !attr->postpone;
}
switch (type) {
case MLX5_INDIRECT_ACTION_TYPE_AGE:
ret = mlx5_hws_age_action_update(priv, idx, update, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_CT:
if (ct_conf->state)
aso = true;
ret = flow_hw_conntrack_update(dev, queue, update, act_idx,
job, push, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
aso = true;
meter_mark = &upd_meter_mark->meter_mark;
/* Find ASO object. */
aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
if (!aso_mtr) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Invalid meter_mark update index");
break;
}
fm = &aso_mtr->fm;
if (upd_meter_mark->profile_valid)
fm->profile = (struct mlx5_flow_meter_profile *)
(meter_mark->profile);
if (upd_meter_mark->color_mode_valid)
fm->color_aware = meter_mark->color_mode;
if (upd_meter_mark->init_color_valid)
aso_mtr->init_color = (meter_mark->color_mode) ?
meter_mark->init_color : RTE_COLOR_GREEN;
if (upd_meter_mark->state_valid)
fm->is_enable = meter_mark->state;
/* Update ASO flow meter by wqe. */
if (mlx5_aso_meter_update_by_wqe(priv->sh, queue,
aso_mtr, &priv->mtr_bulk, job, push)) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Unable to update ASO meter WQE");
break;
}
/* Wait for ASO object completion. */
if (queue == MLX5_HW_INV_QUEUE &&
mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Unable to wait for ASO meter CQE");
}
break;
case MLX5_INDIRECT_ACTION_TYPE_RSS:
ret = flow_dv_action_update(dev, handle, update, error);
break;
default:
ret = -ENOTSUP;
rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"action type not supported");
break;
}
if (job) {
if (ret) {
priv->hw_q[queue].job_idx++;
return ret;
}
job->action = handle;
if (push)
__flow_hw_push_action(dev, queue);
if (aso)
return 0;
rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
priv->hw_q[queue].indir_iq, job);
}
return ret;
}
/**
* Destroy shared action.
*
* @param[in] dev
* Pointer to the rte_eth_dev structure.
* @param[in] queue
* Which queue to be used.
* @param[in] attr
* Operation attribute.
* @param[in] handle
* Action handle to be destroyed.
* @param[in] user_data
* Pointer to the user_data.
* @param[out] error
* Pointer to error structure.
*
* @return
* 0 on success, negative value otherwise and rte_errno is set.
*/
static int
flow_hw_action_handle_destroy(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_op_attr *attr,
struct rte_flow_action_handle *handle,
void *user_data,
struct rte_flow_error *error)
{
uint32_t act_idx = (uint32_t)(uintptr_t)handle;
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
uint32_t idx = act_idx & ((1u << MLX5_INDIRECT_ACTION_TYPE_OFFSET) - 1);
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_aso_mtr_pool *pool = priv->hws_mpool;
struct mlx5_hw_q_job *job = NULL;
struct mlx5_aso_mtr *aso_mtr;
struct mlx5_flow_meter_info *fm;
bool push = true;
bool aso = false;
int ret = 0;
if (attr) {
MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
if (unlikely(!priv->hw_q[queue].job_idx))
return rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Action destroy failed due to queue full.");
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
job->type = MLX5_HW_Q_JOB_TYPE_DESTROY;
job->user_data = user_data;
push = !attr->postpone;
}
switch (type) {
case MLX5_INDIRECT_ACTION_TYPE_AGE:
ret = mlx5_hws_age_action_destroy(priv, age_idx, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_COUNT:
age_idx = mlx5_hws_cnt_age_get(priv->hws_cpool, act_idx);
if (age_idx != 0)
/*
* If this counter belongs to indirect AGE, here is the
* time to update the AGE.
*/
mlx5_hws_age_nb_cnt_decrease(priv, age_idx);
mlx5_hws_cnt_shared_put(priv->hws_cpool, &act_idx);
break;
case MLX5_INDIRECT_ACTION_TYPE_CT:
ret = flow_hw_conntrack_destroy(dev, act_idx, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_METER_MARK:
aso_mtr = mlx5_ipool_get(pool->idx_pool, idx);
if (!aso_mtr) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Invalid meter_mark destroy index");
break;
}
fm = &aso_mtr->fm;
fm->is_enable = 0;
/* Update ASO flow meter by wqe. */
if (mlx5_aso_meter_update_by_wqe(priv->sh, queue, aso_mtr,
&priv->mtr_bulk, job, push)) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Unable to update ASO meter WQE");
break;
}
/* Wait for ASO object completion. */
if (queue == MLX5_HW_INV_QUEUE &&
mlx5_aso_mtr_wait(priv->sh, MLX5_HW_INV_QUEUE, aso_mtr)) {
ret = -EINVAL;
rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Unable to wait for ASO meter CQE");
break;
}
if (!job)
mlx5_ipool_free(pool->idx_pool, idx);
else
aso = true;
break;
case MLX5_INDIRECT_ACTION_TYPE_RSS:
ret = flow_dv_action_destroy(dev, handle, error);
break;
default:
ret = -ENOTSUP;
rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"action type not supported");
break;
}
if (job) {
if (ret) {
priv->hw_q[queue].job_idx++;
return ret;
}
job->action = handle;
if (push)
__flow_hw_push_action(dev, queue);
if (aso)
return ret;
rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
priv->hw_q[queue].indir_iq, job);
}
return ret;
}
static int
flow_hw_query_counter(const struct rte_eth_dev *dev, uint32_t counter,
void *data, struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hws_cnt *cnt;
struct rte_flow_query_count *qc = data;
uint32_t iidx;
uint64_t pkts, bytes;
if (!mlx5_hws_cnt_id_valid(counter))
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"counter are not available");
iidx = mlx5_hws_cnt_iidx(priv->hws_cpool, counter);
cnt = &priv->hws_cpool->pool[iidx];
__hws_cnt_query_raw(priv->hws_cpool, counter, &pkts, &bytes);
qc->hits_set = 1;
qc->bytes_set = 1;
qc->hits = pkts - cnt->reset.hits;
qc->bytes = bytes - cnt->reset.bytes;
if (qc->reset) {
cnt->reset.bytes = bytes;
cnt->reset.hits = pkts;
}
return 0;
}
/**
* Query a flow rule AGE action for aging information.
*
* @param[in] dev
* Pointer to Ethernet device.
* @param[in] age_idx
* Index of AGE action parameter.
* @param[out] data
* Data retrieved by the query.
* @param[out] error
* Perform verbose error reporting if not NULL.
*
* @return
* 0 on success, a negative errno value otherwise and rte_errno is set.
*/
static int
flow_hw_query_age(const struct rte_eth_dev *dev, uint32_t age_idx, void *data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
struct mlx5_indexed_pool *ipool = age_info->ages_ipool;
struct mlx5_hws_age_param *param = mlx5_ipool_get(ipool, age_idx);
struct rte_flow_query_age *resp = data;
if (!param || !param->timeout)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "age data not available");
switch (__atomic_load_n(&param->state, __ATOMIC_RELAXED)) {
case HWS_AGE_AGED_OUT_REPORTED:
case HWS_AGE_AGED_OUT_NOT_REPORTED:
resp->aged = 1;
break;
case HWS_AGE_CANDIDATE:
case HWS_AGE_CANDIDATE_INSIDE_RING:
resp->aged = 0;
break;
case HWS_AGE_FREE:
/*
* When state is FREE the flow itself should be invalid.
* Fall-through.
*/
default:
MLX5_ASSERT(0);
break;
}
resp->sec_since_last_hit_valid = !resp->aged;
if (resp->sec_since_last_hit_valid)
resp->sec_since_last_hit = __atomic_load_n
(&param->sec_since_last_hit, __ATOMIC_RELAXED);
return 0;
}
static int
flow_hw_query(struct rte_eth_dev *dev, struct rte_flow *flow,
const struct rte_flow_action *actions, void *data,
struct rte_flow_error *error)
{
int ret = -EINVAL;
struct rte_flow_hw *hw_flow = (struct rte_flow_hw *)flow;
for (; actions->type != RTE_FLOW_ACTION_TYPE_END; actions++) {
switch (actions->type) {
case RTE_FLOW_ACTION_TYPE_VOID:
break;
case RTE_FLOW_ACTION_TYPE_COUNT:
ret = flow_hw_query_counter(dev, hw_flow->cnt_id, data,
error);
break;
case RTE_FLOW_ACTION_TYPE_AGE:
ret = flow_hw_query_age(dev, hw_flow->age_idx, data,
error);
break;
default:
return rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION,
actions,
"action not supported");
}
}
return ret;
}
/**
* Validate indirect action.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] conf
* Shared action configuration.
* @param[in] action
* Action specification used to create indirect action.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* 0 on success, otherwise negative errno value.
*/
static int
flow_hw_action_validate(struct rte_eth_dev *dev,
const struct rte_flow_indir_action_conf *conf,
const struct rte_flow_action *action,
struct rte_flow_error *err)
{
return flow_hw_action_handle_validate(dev, MLX5_HW_INV_QUEUE, NULL,
conf, action, NULL, err);
}
/**
* Create indirect action.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] conf
* Shared action configuration.
* @param[in] action
* Action specification used to create indirect action.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* A valid shared action handle in case of success, NULL otherwise and
* rte_errno is set.
*/
static struct rte_flow_action_handle *
flow_hw_action_create(struct rte_eth_dev *dev,
const struct rte_flow_indir_action_conf *conf,
const struct rte_flow_action *action,
struct rte_flow_error *err)
{
return flow_hw_action_handle_create(dev, MLX5_HW_INV_QUEUE,
NULL, conf, action, NULL, err);
}
/**
* Destroy the indirect action.
* Release action related resources on the NIC and the memory.
* Lock free, (mutex should be acquired by caller).
* Dispatcher for action type specific call.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] handle
* The indirect action object handle to be removed.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* 0 on success, otherwise negative errno value.
*/
static int
flow_hw_action_destroy(struct rte_eth_dev *dev,
struct rte_flow_action_handle *handle,
struct rte_flow_error *error)
{
return flow_hw_action_handle_destroy(dev, MLX5_HW_INV_QUEUE,
NULL, handle, NULL, error);
}
/**
* Updates in place shared action configuration.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] handle
* The indirect action object handle to be updated.
* @param[in] update
* Action specification used to modify the action pointed by *handle*.
* *update* could be of same type with the action pointed by the *handle*
* handle argument, or some other structures like a wrapper, depending on
* the indirect action type.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* 0 on success, otherwise negative errno value.
*/
static int
flow_hw_action_update(struct rte_eth_dev *dev,
struct rte_flow_action_handle *handle,
const void *update,
struct rte_flow_error *err)
{
return flow_hw_action_handle_update(dev, MLX5_HW_INV_QUEUE,
NULL, handle, update, NULL, err);
}
static int
flow_hw_action_handle_query(struct rte_eth_dev *dev, uint32_t queue,
const struct rte_flow_op_attr *attr,
const struct rte_flow_action_handle *handle,
void *data, void *user_data,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hw_q_job *job = NULL;
uint32_t act_idx = (uint32_t)(uintptr_t)handle;
uint32_t type = act_idx >> MLX5_INDIRECT_ACTION_TYPE_OFFSET;
uint32_t age_idx = act_idx & MLX5_HWS_AGE_IDX_MASK;
int ret;
bool push = true;
bool aso = false;
if (attr) {
MLX5_ASSERT(queue != MLX5_HW_INV_QUEUE);
if (unlikely(!priv->hw_q[queue].job_idx))
return rte_flow_error_set(error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL,
"Action destroy failed due to queue full.");
job = priv->hw_q[queue].job[--priv->hw_q[queue].job_idx];
job->type = MLX5_HW_Q_JOB_TYPE_QUERY;
job->user_data = user_data;
push = !attr->postpone;
}
switch (type) {
case MLX5_INDIRECT_ACTION_TYPE_AGE:
ret = flow_hw_query_age(dev, age_idx, data, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_COUNT:
ret = flow_hw_query_counter(dev, act_idx, data, error);
break;
case MLX5_INDIRECT_ACTION_TYPE_CT:
aso = true;
if (job)
job->profile = (struct rte_flow_action_conntrack *)data;
ret = flow_hw_conntrack_query(dev, queue, act_idx, data,
job, push, error);
break;
default:
ret = -ENOTSUP;
rte_flow_error_set(error, ENOTSUP,
RTE_FLOW_ERROR_TYPE_ACTION, NULL,
"action type not supported");
break;
}
if (job) {
if (ret) {
priv->hw_q[queue].job_idx++;
return ret;
}
job->action = handle;
if (push)
__flow_hw_push_action(dev, queue);
if (aso)
return ret;
rte_ring_enqueue(push ? priv->hw_q[queue].indir_cq :
priv->hw_q[queue].indir_iq, job);
}
return 0;
}
static int
flow_hw_action_query(struct rte_eth_dev *dev,
const struct rte_flow_action_handle *handle, void *data,
struct rte_flow_error *error)
{
return flow_hw_action_handle_query(dev, MLX5_HW_INV_QUEUE, NULL,
handle, data, NULL, error);
}
/**
* Get aged-out flows of a given port on the given HWS flow queue.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] queue_id
* Flow queue to query. Ignored when RTE_FLOW_PORT_FLAG_STRICT_QUEUE not set.
* @param[in, out] contexts
* The address of an array of pointers to the aged-out flows contexts.
* @param[in] nb_contexts
* The length of context array pointers.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* if nb_contexts is 0, return the amount of all aged contexts.
* if nb_contexts is not 0 , return the amount of aged flows reported
* in the context array, otherwise negative errno value.
*/
static int
flow_hw_get_q_aged_flows(struct rte_eth_dev *dev, uint32_t queue_id,
void **contexts, uint32_t nb_contexts,
struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_age_info *age_info = GET_PORT_AGE_INFO(priv);
struct rte_ring *r;
int nb_flows = 0;
if (nb_contexts && !contexts)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "empty context");
if (priv->hws_strict_queue) {
if (queue_id >= age_info->hw_q_age->nb_rings)
return rte_flow_error_set(error, EINVAL,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "invalid queue id");
r = age_info->hw_q_age->aged_lists[queue_id];
} else {
r = age_info->hw_age.aged_list;
MLX5_AGE_SET(age_info, MLX5_AGE_TRIGGER);
}
if (nb_contexts == 0)
return rte_ring_count(r);
while ((uint32_t)nb_flows < nb_contexts) {
uint32_t age_idx;
if (rte_ring_dequeue_elem(r, &age_idx, sizeof(uint32_t)) < 0)
break;
/* get the AGE context if the aged-out index is still valid. */
contexts[nb_flows] = mlx5_hws_age_context_get(priv, age_idx);
if (!contexts[nb_flows])
continue;
nb_flows++;
}
return nb_flows;
}
/**
* Get aged-out flows.
*
* This function is relevant only if RTE_FLOW_PORT_FLAG_STRICT_QUEUE isn't set.
*
* @param[in] dev
* Pointer to the Ethernet device structure.
* @param[in] contexts
* The address of an array of pointers to the aged-out flows contexts.
* @param[in] nb_contexts
* The length of context array pointers.
* @param[out] error
* Perform verbose error reporting if not NULL. Initialized in case of
* error only.
*
* @return
* how many contexts get in success, otherwise negative errno value.
* if nb_contexts is 0, return the amount of all aged contexts.
* if nb_contexts is not 0 , return the amount of aged flows reported
* in the context array.
*/
static int
flow_hw_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
uint32_t nb_contexts, struct rte_flow_error *error)
{
struct mlx5_priv *priv = dev->data->dev_private;
if (priv->hws_strict_queue)
DRV_LOG(WARNING,
"port %u get aged flows called in strict queue mode.",
dev->data->port_id);
return flow_hw_get_q_aged_flows(dev, 0, contexts, nb_contexts, error);
}
const struct mlx5_flow_driver_ops mlx5_flow_hw_drv_ops = {
.info_get = flow_hw_info_get,
.configure = flow_hw_configure,
.pattern_validate = flow_hw_pattern_validate,
.pattern_template_create = flow_hw_pattern_template_create,
.pattern_template_destroy = flow_hw_pattern_template_destroy,
.actions_validate = flow_hw_actions_validate,
.actions_template_create = flow_hw_actions_template_create,
.actions_template_destroy = flow_hw_actions_template_destroy,
.template_table_create = flow_hw_template_table_create,
.template_table_destroy = flow_hw_table_destroy,
.async_flow_create = flow_hw_async_flow_create,
.async_flow_destroy = flow_hw_async_flow_destroy,
.pull = flow_hw_pull,
.push = flow_hw_push,
.async_action_create = flow_hw_action_handle_create,
.async_action_destroy = flow_hw_action_handle_destroy,
.async_action_update = flow_hw_action_handle_update,
.async_action_query = flow_hw_action_handle_query,
.action_validate = flow_hw_action_validate,
.action_create = flow_hw_action_create,
.action_destroy = flow_hw_action_destroy,
.action_update = flow_hw_action_update,
.action_query = flow_hw_action_query,
.query = flow_hw_query,
.get_aged_flows = flow_hw_get_aged_flows,
.get_q_aged_flows = flow_hw_get_q_aged_flows,
};
/**
* Creates a control flow using flow template API on @p proxy_dev device,
* on behalf of @p owner_dev device.
*
* This function uses locks internally to synchronize access to the
* flow queue.
*
* Created flow is stored in private list associated with @p proxy_dev device.
*
* @param owner_dev
* Pointer to Ethernet device on behalf of which flow is created.
* @param proxy_dev
* Pointer to Ethernet device on which flow is created.
* @param table
* Pointer to flow table.
* @param items
* Pointer to flow rule items.
* @param item_template_idx
* Index of an item template associated with @p table.
* @param actions
* Pointer to flow rule actions.
* @param action_template_idx
* Index of an action template associated with @p table.
*
* @return
* 0 on success, negative errno value otherwise and rte_errno set.
*/
static __rte_unused int
flow_hw_create_ctrl_flow(struct rte_eth_dev *owner_dev,
struct rte_eth_dev *proxy_dev,
struct rte_flow_template_table *table,
struct rte_flow_item items[],
uint8_t item_template_idx,
struct rte_flow_action actions[],
uint8_t action_template_idx)
{
struct mlx5_priv *priv = proxy_dev->data->dev_private;
uint32_t queue = CTRL_QUEUE_ID(priv);
struct rte_flow_op_attr op_attr = {
.postpone = 0,
};
struct rte_flow *flow = NULL;
struct mlx5_hw_ctrl_flow *entry = NULL;
int ret;
rte_spinlock_lock(&priv->hw_ctrl_lock);
entry = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_SYS, sizeof(*entry),
0, SOCKET_ID_ANY);
if (!entry) {
DRV_LOG(ERR, "port %u not enough memory to create control flows",
proxy_dev->data->port_id);
rte_errno = ENOMEM;
ret = -rte_errno;
goto error;
}
flow = flow_hw_async_flow_create(proxy_dev, queue, &op_attr, table,
items, item_template_idx,
actions, action_template_idx,
NULL, NULL);
if (!flow) {
DRV_LOG(ERR, "port %u failed to enqueue create control"
" flow operation", proxy_dev->data->port_id);
ret = -rte_errno;
goto error;
}
ret = flow_hw_push(proxy_dev, queue, NULL);
if (ret) {
DRV_LOG(ERR, "port %u failed to drain control flow queue",
proxy_dev->data->port_id);
goto error;
}
ret = __flow_hw_pull_comp(proxy_dev, queue, 1, NULL);
if (ret) {
DRV_LOG(ERR, "port %u failed to insert control flow",
proxy_dev->data->port_id);
rte_errno = EINVAL;
ret = -rte_errno;
goto error;
}
entry->owner_dev = owner_dev;
entry->flow = flow;
LIST_INSERT_HEAD(&priv->hw_ctrl_flows, entry, next);
rte_spinlock_unlock(&priv->hw_ctrl_lock);
return 0;
error:
if (entry)
mlx5_free(entry);
rte_spinlock_unlock(&priv->hw_ctrl_lock);
return ret;
}
/**
* Destroys a control flow @p flow using flow template API on @p dev device.
*
* This function uses locks internally to synchronize access to the
* flow queue.
*
* If the @p flow is stored on any private list/pool, then caller must free up
* the relevant resources.
*
* @param dev
* Pointer to Ethernet device.
* @param flow
* Pointer to flow rule.
*
* @return
* 0 on success, non-zero value otherwise.
*/
static int
flow_hw_destroy_ctrl_flow(struct rte_eth_dev *dev, struct rte_flow *flow)
{
struct mlx5_priv *priv = dev->data->dev_private;
uint32_t queue = CTRL_QUEUE_ID(priv);
struct rte_flow_op_attr op_attr = {
.postpone = 0,
};
int ret;
rte_spinlock_lock(&priv->hw_ctrl_lock);
ret = flow_hw_async_flow_destroy(dev, queue, &op_attr, flow, NULL, NULL);
if (ret) {
DRV_LOG(ERR, "port %u failed to enqueue destroy control"
" flow operation", dev->data->port_id);
goto exit;
}
ret = flow_hw_push(dev, queue, NULL);
if (ret) {
DRV_LOG(ERR, "port %u failed to drain control flow queue",
dev->data->port_id);
goto exit;
}
ret = __flow_hw_pull_comp(dev, queue, 1, NULL);
if (ret) {
DRV_LOG(ERR, "port %u failed to destroy control flow",
dev->data->port_id);
rte_errno = EINVAL;
ret = -rte_errno;
goto exit;
}
exit:
rte_spinlock_unlock(&priv->hw_ctrl_lock);
return ret;
}
/**
* Destroys control flows created on behalf of @p owner device on @p dev device.
*
* @param dev
* Pointer to Ethernet device on which control flows were created.
* @param owner
* Pointer to Ethernet device owning control flows.
*
* @return
* 0 on success, otherwise negative error code is returned and
* rte_errno is set.
*/
static int
flow_hw_flush_ctrl_flows_owned_by(struct rte_eth_dev *dev, struct rte_eth_dev *owner)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hw_ctrl_flow *cf;
struct mlx5_hw_ctrl_flow *cf_next;
int ret;
cf = LIST_FIRST(&priv->hw_ctrl_flows);
while (cf != NULL) {
cf_next = LIST_NEXT(cf, next);
if (cf->owner_dev == owner) {
ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
if (ret) {
rte_errno = ret;
return -ret;
}
LIST_REMOVE(cf, next);
mlx5_free(cf);
}
cf = cf_next;
}
return 0;
}
/**
* Destroys control flows created for @p owner_dev device.
*
* @param owner_dev
* Pointer to Ethernet device owning control flows.
*
* @return
* 0 on success, otherwise negative error code is returned and
* rte_errno is set.
*/
int
mlx5_flow_hw_flush_ctrl_flows(struct rte_eth_dev *owner_dev)
{
struct mlx5_priv *owner_priv = owner_dev->data->dev_private;
struct rte_eth_dev *proxy_dev;
uint16_t owner_port_id = owner_dev->data->port_id;
uint16_t proxy_port_id = owner_dev->data->port_id;
int ret;
/* Flush all flows created by this port for itself. */
ret = flow_hw_flush_ctrl_flows_owned_by(owner_dev, owner_dev);
if (ret)
return ret;
/* Flush all flows created for this port on proxy port. */
if (owner_priv->sh->config.dv_esw_en) {
ret = rte_flow_pick_transfer_proxy(owner_port_id, &proxy_port_id, NULL);
if (ret == -ENODEV) {
DRV_LOG(DEBUG, "Unable to find transfer proxy port for port %u. It was "
"probably closed. Control flows were cleared.",
owner_port_id);
rte_errno = 0;
return 0;
} else if (ret) {
DRV_LOG(ERR, "Unable to find proxy port for port %u (ret = %d)",
owner_port_id, ret);
return ret;
}
proxy_dev = &rte_eth_devices[proxy_port_id];
} else {
proxy_dev = owner_dev;
}
return flow_hw_flush_ctrl_flows_owned_by(proxy_dev, owner_dev);
}
/**
* Destroys all control flows created on @p dev device.
*
* @param owner_dev
* Pointer to Ethernet device.
*
* @return
* 0 on success, otherwise negative error code is returned and
* rte_errno is set.
*/
static int
flow_hw_flush_all_ctrl_flows(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_hw_ctrl_flow *cf;
struct mlx5_hw_ctrl_flow *cf_next;
int ret;
cf = LIST_FIRST(&priv->hw_ctrl_flows);
while (cf != NULL) {
cf_next = LIST_NEXT(cf, next);
ret = flow_hw_destroy_ctrl_flow(dev, cf->flow);
if (ret) {
rte_errno = ret;
return -ret;
}
LIST_REMOVE(cf, next);
mlx5_free(cf);
cf = cf_next;
}
return 0;
}
int
mlx5_flow_hw_esw_create_sq_miss_flow(struct rte_eth_dev *dev, uint32_t sqn)
{
uint16_t port_id = dev->data->port_id;
struct rte_flow_item_ethdev esw_mgr_spec = {
.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
};
struct rte_flow_item_ethdev esw_mgr_mask = {
.port_id = MLX5_REPRESENTED_PORT_ESW_MGR,
};
struct rte_flow_item_tag reg_c0_spec = {
.index = (uint8_t)REG_C_0,
.data = flow_hw_esw_mgr_regc_marker(dev),
};
struct rte_flow_item_tag reg_c0_mask = {
.index = 0xff,
.data = flow_hw_esw_mgr_regc_marker_mask(dev),
};
struct mlx5_rte_flow_item_sq sq_spec = {
.queue = sqn,
};
struct rte_flow_action_ethdev port = {
.port_id = port_id,
};
struct rte_flow_item items[3] = { { 0 } };
struct rte_flow_action actions[3] = { { 0 } };
struct rte_eth_dev *proxy_dev;
struct mlx5_priv *proxy_priv;
uint16_t proxy_port_id = dev->data->port_id;
int ret;
ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
if (ret) {
DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
"port must be present to create default SQ miss flows.",
port_id);
return ret;
}
proxy_dev = &rte_eth_devices[proxy_port_id];
proxy_priv = proxy_dev->data->dev_private;
if (!proxy_priv->dr_ctx) {
DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
"for HWS to create default SQ miss flows. Default flows will "
"not be created.",
proxy_port_id, port_id);
return 0;
}
if (!proxy_priv->hw_esw_sq_miss_root_tbl ||
!proxy_priv->hw_esw_sq_miss_tbl) {
DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
"default flow tables were not created.",
proxy_port_id, port_id);
rte_errno = ENOMEM;
return -rte_errno;
}
/*
* Create a root SQ miss flow rule - match E-Switch Manager and SQ,
* and jump to group 1.
*/
items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.spec = &esw_mgr_spec,
.mask = &esw_mgr_mask,
};
items[1] = (struct rte_flow_item){
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &sq_spec,
};
items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_END,
};
actions[0] = (struct rte_flow_action){
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
};
actions[1] = (struct rte_flow_action){
.type = RTE_FLOW_ACTION_TYPE_JUMP,
};
actions[2] = (struct rte_flow_action) {
.type = RTE_FLOW_ACTION_TYPE_END,
};
ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_root_tbl,
items, 0, actions, 0);
if (ret) {
DRV_LOG(ERR, "Port %u failed to create root SQ miss flow rule for SQ %u, ret %d",
port_id, sqn, ret);
return ret;
}
/*
* Create a non-root SQ miss flow rule - match REG_C_0 marker and SQ,
* and forward to port.
*/
items[0] = (struct rte_flow_item){
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_TAG,
.spec = &reg_c0_spec,
.mask = &reg_c0_mask,
};
items[1] = (struct rte_flow_item){
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &sq_spec,
};
items[2] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_END,
};
actions[0] = (struct rte_flow_action){
.type = RTE_FLOW_ACTION_TYPE_REPRESENTED_PORT,
.conf = &port,
};
actions[1] = (struct rte_flow_action){
.type = RTE_FLOW_ACTION_TYPE_END,
};
ret = flow_hw_create_ctrl_flow(dev, proxy_dev, proxy_priv->hw_esw_sq_miss_tbl,
items, 0, actions, 0);
if (ret) {
DRV_LOG(ERR, "Port %u failed to create HWS SQ miss flow rule for SQ %u, ret %d",
port_id, sqn, ret);
return ret;
}
return 0;
}
int
mlx5_flow_hw_esw_create_default_jump_flow(struct rte_eth_dev *dev)
{
uint16_t port_id = dev->data->port_id;
struct rte_flow_item_ethdev port_spec = {
.port_id = port_id,
};
struct rte_flow_item items[] = {
{
.type = RTE_FLOW_ITEM_TYPE_REPRESENTED_PORT,
.spec = &port_spec,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
struct rte_flow_action_jump jump = {
.group = 1,
};
struct rte_flow_action actions[] = {
{
.type = RTE_FLOW_ACTION_TYPE_JUMP,
.conf = &jump,
},
{
.type = RTE_FLOW_ACTION_TYPE_END,
}
};
struct rte_eth_dev *proxy_dev;
struct mlx5_priv *proxy_priv;
uint16_t proxy_port_id = dev->data->port_id;
int ret;
ret = rte_flow_pick_transfer_proxy(port_id, &proxy_port_id, NULL);
if (ret) {
DRV_LOG(ERR, "Unable to pick transfer proxy port for port %u. Transfer proxy "
"port must be present to create default FDB jump rule.",
port_id);
return ret;
}
proxy_dev = &rte_eth_devices[proxy_port_id];
proxy_priv = proxy_dev->data->dev_private;
if (!proxy_priv->dr_ctx) {
DRV_LOG(DEBUG, "Transfer proxy port (port %u) of port %u must be configured "
"for HWS to create default FDB jump rule. Default rule will "
"not be created.",
proxy_port_id, port_id);
return 0;
}
if (!proxy_priv->hw_esw_zero_tbl) {
DRV_LOG(ERR, "Transfer proxy port (port %u) of port %u was configured, but "
"default flow tables were not created.",
proxy_port_id, port_id);
rte_errno = EINVAL;
return -rte_errno;
}
return flow_hw_create_ctrl_flow(dev, proxy_dev,
proxy_priv->hw_esw_zero_tbl,
items, 0, actions, 0);
}
int
mlx5_flow_hw_create_tx_default_mreg_copy_flow(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_item_eth promisc = {
.dst.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.src.addr_bytes = "\x00\x00\x00\x00\x00\x00",
.type = 0,
};
struct rte_flow_item eth_all[] = {
[0] = {
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = &promisc,
.mask = &promisc,
},
[1] = {
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
struct rte_flow_action_modify_field mreg_action = {
.operation = RTE_FLOW_MODIFY_SET,
.dst = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_C_1,
},
.src = {
.field = (enum rte_flow_field_id)MLX5_RTE_FLOW_FIELD_META_REG,
.level = REG_A,
},
.width = 32,
};
struct rte_flow_action copy_reg_action[] = {
[0] = {
.type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD,
.conf = &mreg_action,
},
[1] = {
.type = RTE_FLOW_ACTION_TYPE_JUMP,
},
[2] = {
.type = RTE_FLOW_ACTION_TYPE_END,
},
};
MLX5_ASSERT(priv->master);
if (!priv->dr_ctx || !priv->hw_tx_meta_cpy_tbl)
return 0;
return flow_hw_create_ctrl_flow(dev, dev,
priv->hw_tx_meta_cpy_tbl,
eth_all, 0, copy_reg_action, 0);
}
int
mlx5_flow_hw_tx_repr_matching_flow(struct rte_eth_dev *dev, uint32_t sqn)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_rte_flow_item_sq sq_spec = {
.queue = sqn,
};
struct rte_flow_item items[] = {
{
.type = (enum rte_flow_item_type)MLX5_RTE_FLOW_ITEM_TYPE_SQ,
.spec = &sq_spec,
},
{
.type = RTE_FLOW_ITEM_TYPE_END,
},
};
/*
* Allocate actions array suitable for all cases - extended metadata enabled or not.
* With extended metadata there will be an additional MODIFY_FIELD action before JUMP.
*/
struct rte_flow_action actions[] = {
{ .type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD },
{ .type = RTE_FLOW_ACTION_TYPE_JUMP },
{ .type = RTE_FLOW_ACTION_TYPE_END },
{ .type = RTE_FLOW_ACTION_TYPE_END },
};
/* It is assumed that caller checked for representor matching. */
MLX5_ASSERT(priv->sh->config.repr_matching);
if (!priv->dr_ctx) {
DRV_LOG(DEBUG, "Port %u must be configured for HWS, before creating "
"default egress flow rules. Omitting creation.",
dev->data->port_id);
return 0;
}
if (!priv->hw_tx_repr_tagging_tbl) {
DRV_LOG(ERR, "Port %u is configured for HWS, but table for default "
"egress flow rules does not exist.",
dev->data->port_id);
rte_errno = EINVAL;
return -rte_errno;
}
/*
* If extended metadata mode is enabled, then an additional MODIFY_FIELD action must be
* placed before terminating JUMP action.
*/
if (priv->sh->config.dv_xmeta_en == MLX5_XMETA_MODE_META32_HWS) {
actions[1].type = RTE_FLOW_ACTION_TYPE_MODIFY_FIELD;
actions[2].type = RTE_FLOW_ACTION_TYPE_JUMP;
}
return flow_hw_create_ctrl_flow(dev, dev, priv->hw_tx_repr_tagging_tbl,
items, 0, actions, 0);
}
static uint32_t
__calc_pattern_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
switch (eth_pattern_type) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
return MLX5_CTRL_PROMISCUOUS;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
return MLX5_CTRL_ALL_MULTICAST;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
return MLX5_CTRL_BROADCAST;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
return MLX5_CTRL_IPV4_MULTICAST;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
return MLX5_CTRL_IPV6_MULTICAST;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
return MLX5_CTRL_DMAC;
default:
/* Should not reach here. */
MLX5_ASSERT(false);
return 0;
}
}
static uint32_t
__calc_vlan_flags(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type)
{
switch (eth_pattern_type) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
return MLX5_CTRL_VLAN_FILTER;
default:
return 0;
}
}
static bool
eth_pattern_type_is_requested(const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type,
uint32_t flags)
{
uint32_t pattern_flags = __calc_pattern_flags(eth_pattern_type);
uint32_t vlan_flags = __calc_vlan_flags(eth_pattern_type);
bool pattern_requested = !!(pattern_flags & flags);
bool consider_vlan = vlan_flags || (MLX5_CTRL_VLAN_FILTER & flags);
bool vlan_requested = !!(vlan_flags & flags);
if (consider_vlan)
return pattern_requested && vlan_requested;
else
return pattern_requested;
}
static bool
rss_type_is_requested(struct mlx5_priv *priv,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct rte_flow_actions_template *at = priv->hw_ctrl_rx->rss[rss_type];
unsigned int i;
for (i = 0; at->actions[i].type != RTE_FLOW_ACTION_TYPE_END; ++i) {
if (at->actions[i].type == RTE_FLOW_ACTION_TYPE_RSS) {
const struct rte_flow_action_rss *rss = at->actions[i].conf;
uint64_t rss_types = rss->types;
if ((rss_types & priv->rss_conf.rss_hf) != rss_types)
return false;
}
}
return true;
}
static const struct rte_flow_item_eth *
__get_eth_spec(const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern)
{
switch (pattern) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
return &ctrl_rx_eth_promisc_spec;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
return &ctrl_rx_eth_mcast_spec;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
return &ctrl_rx_eth_bcast_spec;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
return &ctrl_rx_eth_ipv4_mcast_spec;
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
return &ctrl_rx_eth_ipv6_mcast_spec;
default:
/* This case should not be reached. */
MLX5_ASSERT(false);
return NULL;
}
}
static int
__flow_hw_ctrl_flows_single(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
struct rte_flow_item items[5];
struct rte_flow_action actions[] = {
{ .type = RTE_FLOW_ACTION_TYPE_RSS },
{ .type = RTE_FLOW_ACTION_TYPE_END },
};
if (!eth_spec)
return -EINVAL;
memset(items, 0, sizeof(items));
items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = eth_spec,
};
items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
/* Without VLAN filtering, only a single flow rule must be created. */
return flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0);
}
static int
__flow_hw_ctrl_flows_single_vlan(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct mlx5_priv *priv = dev->data->dev_private;
const struct rte_flow_item_eth *eth_spec = __get_eth_spec(pattern_type);
struct rte_flow_item items[5];
struct rte_flow_action actions[] = {
{ .type = RTE_FLOW_ACTION_TYPE_RSS },
{ .type = RTE_FLOW_ACTION_TYPE_END },
};
unsigned int i;
if (!eth_spec)
return -EINVAL;
memset(items, 0, sizeof(items));
items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = eth_spec,
};
/* Optional VLAN for now will be VOID - will be filled later. */
items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
/* Since VLAN filtering is done, create a single flow rule for each registered vid. */
for (i = 0; i < priv->vlan_filter_n; ++i) {
uint16_t vlan = priv->vlan_filter[i];
struct rte_flow_item_vlan vlan_spec = {
.tci = rte_cpu_to_be_16(vlan),
};
items[1].spec = &vlan_spec;
if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
return -rte_errno;
}
return 0;
}
static int
__flow_hw_ctrl_flows_unicast(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct rte_flow_item_eth eth_spec;
struct rte_flow_item items[5];
struct rte_flow_action actions[] = {
{ .type = RTE_FLOW_ACTION_TYPE_RSS },
{ .type = RTE_FLOW_ACTION_TYPE_END },
};
const struct rte_ether_addr cmp = {
.addr_bytes = "\x00\x00\x00\x00\x00\x00",
};
unsigned int i;
RTE_SET_USED(pattern_type);
memset(&eth_spec, 0, sizeof(eth_spec));
memset(items, 0, sizeof(items));
items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = &eth_spec,
};
items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VOID };
items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
struct rte_ether_addr *mac = &dev->data->mac_addrs[i];
if (!memcmp(mac, &cmp, sizeof(*mac)))
continue;
memcpy(&eth_spec.dst.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
return -rte_errno;
}
return 0;
}
static int
__flow_hw_ctrl_flows_unicast_vlan(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct rte_flow_item_eth eth_spec;
struct rte_flow_item items[5];
struct rte_flow_action actions[] = {
{ .type = RTE_FLOW_ACTION_TYPE_RSS },
{ .type = RTE_FLOW_ACTION_TYPE_END },
};
const struct rte_ether_addr cmp = {
.addr_bytes = "\x00\x00\x00\x00\x00\x00",
};
unsigned int i;
unsigned int j;
RTE_SET_USED(pattern_type);
memset(&eth_spec, 0, sizeof(eth_spec));
memset(items, 0, sizeof(items));
items[0] = (struct rte_flow_item){
.type = RTE_FLOW_ITEM_TYPE_ETH,
.spec = &eth_spec,
};
items[1] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_VLAN };
items[2] = flow_hw_get_ctrl_rx_l3_item(rss_type);
items[3] = flow_hw_get_ctrl_rx_l4_item(rss_type);
items[4] = (struct rte_flow_item){ .type = RTE_FLOW_ITEM_TYPE_END };
for (i = 0; i < MLX5_MAX_MAC_ADDRESSES; ++i) {
struct rte_ether_addr *mac = &dev->data->mac_addrs[i];
if (!memcmp(mac, &cmp, sizeof(*mac)))
continue;
memcpy(&eth_spec.dst.addr_bytes, mac->addr_bytes, RTE_ETHER_ADDR_LEN);
for (j = 0; j < priv->vlan_filter_n; ++j) {
uint16_t vlan = priv->vlan_filter[j];
struct rte_flow_item_vlan vlan_spec = {
.tci = rte_cpu_to_be_16(vlan),
};
items[1].spec = &vlan_spec;
if (flow_hw_create_ctrl_flow(dev, dev, tbl, items, 0, actions, 0))
return -rte_errno;
}
}
return 0;
}
static int
__flow_hw_ctrl_flows(struct rte_eth_dev *dev,
struct rte_flow_template_table *tbl,
const enum mlx5_flow_ctrl_rx_eth_pattern_type pattern_type,
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type)
{
switch (pattern_type) {
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_ALL_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST:
return __flow_hw_ctrl_flows_single(dev, tbl, pattern_type, rss_type);
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_BCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV4_MCAST_VLAN:
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_IPV6_MCAST_VLAN:
return __flow_hw_ctrl_flows_single_vlan(dev, tbl, pattern_type, rss_type);
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC:
return __flow_hw_ctrl_flows_unicast(dev, tbl, pattern_type, rss_type);
case MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_DMAC_VLAN:
return __flow_hw_ctrl_flows_unicast_vlan(dev, tbl, pattern_type, rss_type);
default:
/* Should not reach here. */
MLX5_ASSERT(false);
rte_errno = EINVAL;
return -EINVAL;
}
}
int
mlx5_flow_hw_ctrl_flows(struct rte_eth_dev *dev, uint32_t flags)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_flow_hw_ctrl_rx *hw_ctrl_rx;
unsigned int i;
unsigned int j;
int ret = 0;
RTE_SET_USED(priv);
RTE_SET_USED(flags);
if (!priv->dr_ctx) {
DRV_LOG(DEBUG, "port %u Control flow rules will not be created. "
"HWS needs to be configured beforehand.",
dev->data->port_id);
return 0;
}
if (!priv->hw_ctrl_rx) {
DRV_LOG(ERR, "port %u Control flow rules templates were not created.",
dev->data->port_id);
rte_errno = EINVAL;
return -rte_errno;
}
hw_ctrl_rx = priv->hw_ctrl_rx;
for (i = 0; i < MLX5_FLOW_HW_CTRL_RX_ETH_PATTERN_MAX; ++i) {
const enum mlx5_flow_ctrl_rx_eth_pattern_type eth_pattern_type = i;
if (!eth_pattern_type_is_requested(eth_pattern_type, flags))
continue;
for (j = 0; j < MLX5_FLOW_HW_CTRL_RX_EXPANDED_RSS_MAX; ++j) {
const enum mlx5_flow_ctrl_rx_expanded_rss_type rss_type = j;
struct rte_flow_actions_template *at;
struct mlx5_flow_hw_ctrl_rx_table *tmpls = &hw_ctrl_rx->tables[i][j];
const struct mlx5_flow_template_table_cfg cfg = {
.attr = tmpls->attr,
.external = 0,
};
if (!hw_ctrl_rx->rss[rss_type]) {
at = flow_hw_create_ctrl_rx_rss_template(dev, rss_type);
if (!at)
return -rte_errno;
hw_ctrl_rx->rss[rss_type] = at;
} else {
at = hw_ctrl_rx->rss[rss_type];
}
if (!rss_type_is_requested(priv, rss_type))
continue;
if (!tmpls->tbl) {
tmpls->tbl = flow_hw_table_create(dev, &cfg,
&tmpls->pt, 1, &at, 1, NULL);
if (!tmpls->tbl) {
DRV_LOG(ERR, "port %u Failed to create template table "
"for control flow rules. Unable to create "
"control flow rules.",
dev->data->port_id);
return -rte_errno;
}
}
ret = __flow_hw_ctrl_flows(dev, tmpls->tbl, eth_pattern_type, rss_type);
if (ret) {
DRV_LOG(ERR, "port %u Failed to create control flow rule.",
dev->data->port_id);
return ret;
}
}
}
return 0;
}
void
mlx5_flow_meter_uninit(struct rte_eth_dev *dev)
{
struct mlx5_priv *priv = dev->data->dev_private;
if (priv->mtr_policy_arr) {
mlx5_free(priv->mtr_policy_arr);
priv->mtr_policy_arr = NULL;
}
if (priv->mtr_profile_arr) {
mlx5_free(priv->mtr_profile_arr);
priv->mtr_profile_arr = NULL;
}
if (priv->hws_mpool) {
mlx5_aso_mtr_queue_uninit(priv->sh, priv->hws_mpool, NULL);
mlx5_ipool_destroy(priv->hws_mpool->idx_pool);
mlx5_free(priv->hws_mpool);
priv->hws_mpool = NULL;
}
if (priv->mtr_bulk.aso) {
mlx5_free(priv->mtr_bulk.aso);
priv->mtr_bulk.aso = NULL;
priv->mtr_bulk.size = 0;
mlx5_aso_queue_uninit(priv->sh, ASO_OPC_MOD_POLICER);
}
if (priv->mtr_bulk.action) {
mlx5dr_action_destroy(priv->mtr_bulk.action);
priv->mtr_bulk.action = NULL;
}
if (priv->mtr_bulk.devx_obj) {
claim_zero(mlx5_devx_cmd_destroy(priv->mtr_bulk.devx_obj));
priv->mtr_bulk.devx_obj = NULL;
}
}
int
mlx5_flow_meter_init(struct rte_eth_dev *dev,
uint32_t nb_meters,
uint32_t nb_meter_profiles,
uint32_t nb_meter_policies,
uint32_t nb_queues)
{
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_devx_obj *dcs = NULL;
uint32_t log_obj_size;
int ret = 0;
int reg_id;
struct mlx5_aso_mtr *aso;
uint32_t i;
struct rte_flow_error error;
uint32_t flags;
uint32_t nb_mtrs = rte_align32pow2(nb_meters);
struct mlx5_indexed_pool_config cfg = {
.size = sizeof(struct mlx5_aso_mtr),
.trunk_size = 1 << 12,
.per_core_cache = 1 << 13,
.need_lock = 1,
.release_mem_en = !!priv->sh->config.reclaim_mode,
.malloc = mlx5_malloc,
.max_idx = nb_meters,
.free = mlx5_free,
.type = "mlx5_hw_mtr_mark_action",
};
if (!nb_meters || !nb_meter_profiles || !nb_meter_policies) {
ret = ENOTSUP;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter configuration is invalid.");
goto err;
}
if (!priv->mtr_en || !priv->sh->meter_aso_en) {
ret = ENOTSUP;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter ASO is not supported.");
goto err;
}
priv->mtr_config.nb_meters = nb_meters;
log_obj_size = rte_log2_u32(nb_meters >> 1);
dcs = mlx5_devx_cmd_create_flow_meter_aso_obj
(priv->sh->cdev->ctx, priv->sh->cdev->pdn,
log_obj_size);
if (!dcs) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter ASO object allocation failed.");
goto err;
}
priv->mtr_bulk.devx_obj = dcs;
reg_id = mlx5_flow_get_reg_id(dev, MLX5_MTR_COLOR, 0, NULL);
if (reg_id < 0) {
ret = ENOTSUP;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter register is not available.");
goto err;
}
flags = MLX5DR_ACTION_FLAG_HWS_RX | MLX5DR_ACTION_FLAG_HWS_TX;
if (priv->sh->config.dv_esw_en && priv->master)
flags |= MLX5DR_ACTION_FLAG_HWS_FDB;
priv->mtr_bulk.action = mlx5dr_action_create_aso_meter
(priv->dr_ctx, (struct mlx5dr_devx_obj *)dcs,
reg_id - REG_C_0, flags);
if (!priv->mtr_bulk.action) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter action creation failed.");
goto err;
}
priv->mtr_bulk.aso = mlx5_malloc(MLX5_MEM_ZERO,
sizeof(struct mlx5_aso_mtr) *
nb_meters,
RTE_CACHE_LINE_SIZE,
SOCKET_ID_ANY);
if (!priv->mtr_bulk.aso) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter bulk ASO allocation failed.");
goto err;
}
priv->mtr_bulk.size = nb_meters;
aso = priv->mtr_bulk.aso;
for (i = 0; i < priv->mtr_bulk.size; i++) {
aso->type = ASO_METER_DIRECT;
aso->state = ASO_METER_WAIT;
aso->offset = i;
aso++;
}
priv->hws_mpool = mlx5_malloc(MLX5_MEM_ZERO,
sizeof(struct mlx5_aso_mtr_pool),
RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (!priv->hws_mpool) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter ipool allocation failed.");
goto err;
}
priv->hws_mpool->devx_obj = priv->mtr_bulk.devx_obj;
priv->hws_mpool->action = priv->mtr_bulk.action;
priv->hws_mpool->nb_sq = nb_queues;
if (mlx5_aso_mtr_queue_init(priv->sh, priv->hws_mpool,
&priv->sh->mtrmng->pools_mng, nb_queues)) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter ASO queue allocation failed.");
goto err;
}
/*
* No need for local cache if Meter number is a small number.
* Since flow insertion rate will be very limited in that case.
* Here let's set the number to less than default trunk size 4K.
*/
if (nb_mtrs <= cfg.trunk_size) {
cfg.per_core_cache = 0;
cfg.trunk_size = nb_mtrs;
} else if (nb_mtrs <= MLX5_HW_IPOOL_SIZE_THRESHOLD) {
cfg.per_core_cache = MLX5_HW_IPOOL_CACHE_MIN;
}
priv->hws_mpool->idx_pool = mlx5_ipool_create(&cfg);
priv->mtr_config.nb_meter_profiles = nb_meter_profiles;
priv->mtr_profile_arr =
mlx5_malloc(MLX5_MEM_ZERO,
sizeof(struct mlx5_flow_meter_profile) *
nb_meter_profiles,
RTE_CACHE_LINE_SIZE,
SOCKET_ID_ANY);
if (!priv->mtr_profile_arr) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter profile allocation failed.");
goto err;
}
priv->mtr_config.nb_meter_policies = nb_meter_policies;
priv->mtr_policy_arr =
mlx5_malloc(MLX5_MEM_ZERO,
sizeof(struct mlx5_flow_meter_policy) *
nb_meter_policies,
RTE_CACHE_LINE_SIZE,
SOCKET_ID_ANY);
if (!priv->mtr_policy_arr) {
ret = ENOMEM;
rte_flow_error_set(&error, ENOMEM,
RTE_FLOW_ERROR_TYPE_UNSPECIFIED,
NULL, "Meter policy allocation failed.");
goto err;
}
return 0;
err:
mlx5_flow_meter_uninit(dev);
return ret;
}
#endif
Reference in New Issue View Git Blame Copy Permalink