net/mlx5: support modify header using Direct Verbs

This patch implements the set of actions to support offload
of packet header modifications to MLX5 NIC.

Implementation is based on RFC [1].

[1] http://mails.dpdk.org/archives/dev/2018-November/119971.html

Signed-off-by: Dekel Peled <dekelp@mellanox.com>
Acked-by: Shahaf Shuler <shahafs@mellanox.com>

drivers/net/mlx5/Makefile

@@ -8,7 +8,7 @@ include $(RTE_SDK)/mk/rte.vars.mk
 LIB = librte_pmd_mlx5.a
 LIB_GLUE = $(LIB_GLUE_BASE).$(LIB_GLUE_VERSION)
 LIB_GLUE_BASE = librte_pmd_mlx5_glue.so
-LIB_GLUE_VERSION = 18.11.0
+LIB_GLUE_VERSION = 19.02.0
 
 # Sources.
 SRCS-$(CONFIG_RTE_LIBRTE_MLX5_PMD) += mlx5.c

drivers/net/mlx5/meson.build

@@ -4,7 +4,7 @@
 
 pmd_dlopen = get_option('enable_driver_mlx_glue')
 LIB_GLUE_BASE = 'librte_pmd_mlx5_glue.so'
-LIB_GLUE_VERSION = '18.11.0'
+LIB_GLUE_VERSION = '19.02.0'
 LIB_GLUE = LIB_GLUE_BASE + '.' + LIB_GLUE_VERSION
 if pmd_dlopen
 	dpdk_conf.set('RTE_LIBRTE_MLX5_DLOPEN_DEPS', 1)

drivers/net/mlx5/mlx5.h

@@ -227,6 +227,7 @@ struct priv {
 	LIST_HEAD(ind_tables, mlx5_ind_table_ibv) ind_tbls;
 	LIST_HEAD(matchers, mlx5_flow_dv_matcher) matchers;
 	LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps;
+	LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
 	uint32_t link_speed_capa; /* Link speed capabilities. */
 	struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */
 	struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */

drivers/net/mlx5/mlx5_flow.h

@@ -69,6 +69,18 @@
 	(MLX5_FLOW_LAYER_INNER_L2 | MLX5_FLOW_LAYER_INNER_L3 | \
 	 MLX5_FLOW_LAYER_INNER_L4)
 
+/* Layer Masks. */
+#define MLX5_FLOW_LAYER_L2 \
+	(MLX5_FLOW_LAYER_OUTER_L2 | MLX5_FLOW_LAYER_INNER_L2)
+#define MLX5_FLOW_LAYER_L3_IPV4 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV4 | MLX5_FLOW_LAYER_INNER_L3_IPV4)
+#define MLX5_FLOW_LAYER_L3_IPV6 \
+	(MLX5_FLOW_LAYER_OUTER_L3_IPV6 | MLX5_FLOW_LAYER_INNER_L3_IPV6)
+#define MLX5_FLOW_LAYER_L3 \
+	(MLX5_FLOW_LAYER_L3_IPV4 | MLX5_FLOW_LAYER_L3_IPV6)
+#define MLX5_FLOW_LAYER_L4 \
+	(MLX5_FLOW_LAYER_OUTER_L4 | MLX5_FLOW_LAYER_INNER_L4)
+
 /* Actions */
 #define MLX5_FLOW_ACTION_DROP (1u << 0)
 #define MLX5_FLOW_ACTION_QUEUE (1u << 1)
@@ -110,6 +122,17 @@
 				 MLX5_FLOW_ACTION_NVGRE_DECAP | \
 				 MLX5_FLOW_ACTION_RAW_DECAP)
 
+#define MLX5_FLOW_MODIFY_HDR_ACTIONS (MLX5_FLOW_ACTION_SET_IPV4_SRC | \
+				      MLX5_FLOW_ACTION_SET_IPV4_DST | \
+				      MLX5_FLOW_ACTION_SET_IPV6_SRC | \
+				      MLX5_FLOW_ACTION_SET_IPV6_DST | \
+				      MLX5_FLOW_ACTION_SET_TP_SRC | \
+				      MLX5_FLOW_ACTION_SET_TP_DST | \
+				      MLX5_FLOW_ACTION_SET_TTL | \
+				      MLX5_FLOW_ACTION_DEC_TTL | \
+				      MLX5_FLOW_ACTION_SET_MAC_SRC | \
+				      MLX5_FLOW_ACTION_SET_MAC_DST)
+
 #ifndef IPPROTO_MPLS
 #define IPPROTO_MPLS 137
 #endif
@@ -153,9 +176,6 @@
 /* IBV hash source bits  for IPV6. */
 #define MLX5_IPV6_IBV_RX_HASH (IBV_RX_HASH_SRC_IPV6 | IBV_RX_HASH_DST_IPV6)
 
-/* Max number of actions per DV flow. */
-#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
-
 enum mlx5_flow_drv_type {
 	MLX5_FLOW_TYPE_MIN,
 	MLX5_FLOW_TYPE_DV,
@@ -172,9 +192,6 @@ struct mlx5_flow_dv_match_params {
 	/**< Matcher value. This value is used as the mask or as a key. */
 };
 
-#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
-#define MLX5_ENCAP_MAX_LEN 132
-
 /* Matcher structure. */
 struct mlx5_flow_dv_matcher {
 	LIST_ENTRY(mlx5_flow_dv_matcher) next;
@@ -187,6 +204,8 @@ struct mlx5_flow_dv_matcher {
 	struct mlx5_flow_dv_match_params mask; /**< Matcher mask. */
 };
 
+#define MLX5_ENCAP_MAX_LEN 132
+
 /* Encap/decap resource structure. */
 struct mlx5_flow_dv_encap_decap_resource {
 	LIST_ENTRY(mlx5_flow_dv_encap_decap_resource) next;
@@ -200,6 +219,29 @@ struct mlx5_flow_dv_encap_decap_resource {
 	uint8_t ft_type;
 };
 
+/* Number of modification commands. */
+#define MLX5_MODIFY_NUM 8
+
+/* Modify resource structure */
+struct mlx5_flow_dv_modify_hdr_resource {
+	LIST_ENTRY(mlx5_flow_dv_modify_hdr_resource) next;
+	/* Pointer to next element. */
+	rte_atomic32_t refcnt; /**< Reference counter. */
+	struct ibv_flow_action *verbs_action;
+	/**< Verbs modify header action object. */
+	uint8_t ft_type; /**< Flow table type, Rx or Tx. */
+	uint32_t actions_num; /**< Number of modification actions. */
+	struct mlx5_modification_cmd actions[MLX5_MODIFY_NUM];
+	/**< Modification actions. */
+};
+
+/*
+ * Max number of actions per DV flow.
+ * See CREATE_FLOW_MAX_FLOW_ACTIONS_SUPPORTED
+ * In rdma-core file providers/mlx5/verbs.c
+ */
+#define MLX5_DV_MAX_NUMBER_OF_ACTIONS 8
+
 /* DV flows structure. */
 struct mlx5_flow_dv {
 	uint64_t hash_fields; /**< Fields that participate in the hash. */
@@ -210,6 +252,8 @@ struct mlx5_flow_dv {
 	/**< Holds the value that the packet is compared to. */
 	struct mlx5_flow_dv_encap_decap_resource *encap_decap;
 	/**< Pointer to encap/decap resource in cache. */
+	struct mlx5_flow_dv_modify_hdr_resource *modify_hdr;
+	/**< Pointer to modify header resource in cache. */
 	struct ibv_flow *flow; /**< Installed flow. */
 #ifdef HAVE_IBV_FLOW_DV_SUPPORT
 	struct mlx5dv_flow_action_attr actions[MLX5_DV_MAX_NUMBER_OF_ACTIONS];

drivers/net/mlx5/mlx5_glue.c

@@ -479,6 +479,26 @@ mlx5_glue_dv_create_flow_action_packet_reformat
 #endif
 }
 
+static struct ibv_flow_action *
+mlx5_glue_dv_create_flow_action_modify_header
+					(struct ibv_context *ctx,
+					 size_t actions_sz,
+					 uint64_t actions[],
+					 enum mlx5dv_flow_table_type ft_type)
+{
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+	return mlx5dv_create_flow_action_modify_header(ctx, actions_sz,
+						       actions, ft_type);
+#else
+	(void)ctx;
+	(void)actions_sz;
+	(void)actions;
+	(void)ft_type;
+	return NULL;
+#endif
+}
+
+
 alignas(RTE_CACHE_LINE_SIZE)
 const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.version = MLX5_GLUE_VERSION,
@@ -535,4 +555,6 @@ const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue){
 	.dv_create_flow = mlx5_glue_dv_create_flow,
 	.dv_create_flow_action_packet_reformat =
 			mlx5_glue_dv_create_flow_action_packet_reformat,
+	.dv_create_flow_action_modify_header =
+			mlx5_glue_dv_create_flow_action_modify_header,
 };

drivers/net/mlx5/mlx5_glue.h

@@ -164,6 +164,11 @@ struct mlx5_glue {
 		 void *data,
 		 enum mlx5dv_flow_action_packet_reformat_type reformat_type,
 		 enum mlx5dv_flow_table_type ft_type);
+	struct ibv_flow_action *(*dv_create_flow_action_modify_header)
+					(struct ibv_context *ctx,
+					 size_t actions_sz,
+					 uint64_t actions[],
+					 enum mlx5dv_flow_table_type ft_type);
 };
 
 const struct mlx5_glue *mlx5_glue;

drivers/net/mlx5/mlx5_prm.h

@@ -280,8 +280,14 @@ struct mlx5_cqe {
 /* CQE format value. */
 #define MLX5_COMPRESSED 0x3
 
+/* Write a specific data value to a field. */
+#define MLX5_MODIFICATION_TYPE_SET 1
+
+/* Add a specific data value to a field. */
+#define MLX5_MODIFICATION_TYPE_ADD 2
+
 /* The field of packet to be modified. */
-enum mlx5_modificaiton_field {
+enum mlx5_modification_field {
 	MLX5_MODI_OUT_SMAC_47_16 = 1,
 	MLX5_MODI_OUT_SMAC_15_0,
 	MLX5_MODI_OUT_ETHERTYPE,
@@ -337,23 +343,17 @@ struct mlx5_modification_cmd {
 	union {
 		uint32_t data0;
 		struct {
-			unsigned int bits:5;
+			unsigned int length:5;
 			unsigned int rsvd0:3;
-			unsigned int src_offset:5; /* Start bit offset. */
+			unsigned int offset:5;
 			unsigned int rsvd1:3;
-			unsigned int src_field:12;
+			unsigned int field:12;
-			unsigned int type:4;
+			unsigned int action_type:4;
 		};
 	};
 	union {
 		uint32_t data1;
 		uint8_t data[4];
-		struct {
-			unsigned int rsvd2:8;
-			unsigned int dst_offset:8;
-			unsigned int dst_field:12;
-			unsigned int rsvd3:4;
-		};
 	};
 };