/* SPDX-License-Identifier: BSD-3-Clause * Copyright (c) 2021 NVIDIA Corporation & Affiliates */ #include #include #include #include "mlx5.h" #include "mlx5_flow.h" static_assert(sizeof(uint32_t) * CHAR_BIT >= MLX5_PORT_FLEX_ITEM_NUM, "Flex item maximal number exceeds uint32_t bit width"); /** * Routine called once on port initialization to init flex item * related infrastructure initialization * * @param dev * Ethernet device to perform flex item initialization * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flex_item_port_init(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; rte_spinlock_init(&priv->flex_item_sl); MLX5_ASSERT(!priv->flex_item_map); return 0; } /** * Routine called once on port close to perform flex item * related infrastructure cleanup. * * @param dev * Ethernet device to perform cleanup */ void mlx5_flex_item_port_cleanup(struct rte_eth_dev *dev) { struct mlx5_priv *priv = dev->data->dev_private; uint32_t i; for (i = 0; i < MLX5_PORT_FLEX_ITEM_NUM && priv->flex_item_map ; i++) { if (priv->flex_item_map & (1 << i)) { struct mlx5_flex_item *flex = &priv->flex_item[i]; claim_zero(mlx5_list_unregister (priv->sh->flex_parsers_dv, &flex->devx_fp->entry)); flex->devx_fp = NULL; flex->refcnt = 0; priv->flex_item_map &= ~(1 << i); } } } static int mlx5_flex_index(struct mlx5_priv *priv, struct mlx5_flex_item *item) { uintptr_t start = (uintptr_t)&priv->flex_item[0]; uintptr_t entry = (uintptr_t)item; uintptr_t idx = (entry - start) / sizeof(struct mlx5_flex_item); if (entry < start || idx >= MLX5_PORT_FLEX_ITEM_NUM || (entry - start) % sizeof(struct mlx5_flex_item) || !(priv->flex_item_map & (1u << idx))) return -1; return (int)idx; } static struct mlx5_flex_item * mlx5_flex_alloc(struct mlx5_priv *priv) { struct mlx5_flex_item *item = NULL; rte_spinlock_lock(&priv->flex_item_sl); if (~priv->flex_item_map) { uint32_t idx = rte_bsf32(~priv->flex_item_map); if (idx < MLX5_PORT_FLEX_ITEM_NUM) { item = &priv->flex_item[idx]; MLX5_ASSERT(!item->refcnt); MLX5_ASSERT(!item->devx_fp); item->devx_fp = NULL; __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE); priv->flex_item_map |= 1u << idx; } } rte_spinlock_unlock(&priv->flex_item_sl); return item; } static void mlx5_flex_free(struct mlx5_priv *priv, struct mlx5_flex_item *item) { int idx = mlx5_flex_index(priv, item); MLX5_ASSERT(idx >= 0 && idx < MLX5_PORT_FLEX_ITEM_NUM && (priv->flex_item_map & (1u << idx))); if (idx >= 0) { rte_spinlock_lock(&priv->flex_item_sl); MLX5_ASSERT(!item->refcnt); MLX5_ASSERT(!item->devx_fp); item->devx_fp = NULL; __atomic_store_n(&item->refcnt, 0, __ATOMIC_RELEASE); priv->flex_item_map &= ~(1u << idx); rte_spinlock_unlock(&priv->flex_item_sl); } } static uint32_t mlx5_flex_get_bitfield(const struct rte_flow_item_flex *item, uint32_t pos, uint32_t width, uint32_t shift) { const uint8_t *ptr = item->pattern + pos / CHAR_BIT; uint32_t val, vbits; /* Proceed the bitfield start byte. */ MLX5_ASSERT(width <= sizeof(uint32_t) * CHAR_BIT && width); MLX5_ASSERT(width + shift <= sizeof(uint32_t) * CHAR_BIT); if (item->length <= pos / CHAR_BIT) return 0; val = *ptr++ >> (pos % CHAR_BIT); vbits = CHAR_BIT - pos % CHAR_BIT; pos = (pos + vbits) / CHAR_BIT; vbits = RTE_MIN(vbits, width); val &= RTE_BIT32(vbits) - 1; while (vbits < width && pos < item->length) { uint32_t part = RTE_MIN(width - vbits, (uint32_t)CHAR_BIT); uint32_t tmp = *ptr++; pos++; tmp &= RTE_BIT32(part) - 1; val |= tmp << vbits; vbits += part; } return rte_bswap32(val <<= shift); } #define SET_FP_MATCH_SAMPLE_ID(x, def, msk, val, sid) \ do { \ uint32_t tmp, out = (def); \ tmp = MLX5_GET(fte_match_set_misc4, misc4_v, \ prog_sample_field_value_##x); \ tmp = (tmp & ~out) | (val); \ MLX5_SET(fte_match_set_misc4, misc4_v, \ prog_sample_field_value_##x, tmp); \ tmp = MLX5_GET(fte_match_set_misc4, misc4_m, \ prog_sample_field_value_##x); \ tmp = (tmp & ~out) | (msk); \ MLX5_SET(fte_match_set_misc4, misc4_m, \ prog_sample_field_value_##x, tmp); \ tmp = tmp ? (sid) : 0; \ MLX5_SET(fte_match_set_misc4, misc4_v, \ prog_sample_field_id_##x, tmp);\ MLX5_SET(fte_match_set_misc4, misc4_m, \ prog_sample_field_id_##x, tmp); \ } while (0) __rte_always_inline static void mlx5_flex_set_match_sample(void *misc4_m, void *misc4_v, uint32_t def, uint32_t mask, uint32_t value, uint32_t sample_id, uint32_t id) { switch (id) { case 0: SET_FP_MATCH_SAMPLE_ID(0, def, mask, value, sample_id); break; case 1: SET_FP_MATCH_SAMPLE_ID(1, def, mask, value, sample_id); break; case 2: SET_FP_MATCH_SAMPLE_ID(2, def, mask, value, sample_id); break; case 3: SET_FP_MATCH_SAMPLE_ID(3, def, mask, value, sample_id); break; case 4: SET_FP_MATCH_SAMPLE_ID(4, def, mask, value, sample_id); break; case 5: SET_FP_MATCH_SAMPLE_ID(5, def, mask, value, sample_id); break; case 6: SET_FP_MATCH_SAMPLE_ID(6, def, mask, value, sample_id); break; case 7: SET_FP_MATCH_SAMPLE_ID(7, def, mask, value, sample_id); break; default: MLX5_ASSERT(false); break; } #undef SET_FP_MATCH_SAMPLE_ID } /** * Translate item pattern into matcher fields according to translation * array. * * @param dev * Ethernet device to translate flex item on. * @param[in, out] matcher * Flow matcher to configure * @param[in, out] key * Flow matcher value. * @param[in] item * Flow pattern to translate. * @param[in] is_inner * Inner Flex Item (follows after tunnel header). * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ void mlx5_flex_flow_translate_item(struct rte_eth_dev *dev, void *matcher, void *key, const struct rte_flow_item *item, bool is_inner) { const struct rte_flow_item_flex *spec, *mask; void *misc4_m = MLX5_ADDR_OF(fte_match_param, matcher, misc_parameters_4); void *misc4_v = MLX5_ADDR_OF(fte_match_param, key, misc_parameters_4); struct mlx5_flex_item *tp; uint32_t i, pos = 0; RTE_SET_USED(dev); MLX5_ASSERT(item->spec && item->mask); spec = item->spec; mask = item->mask; tp = (struct mlx5_flex_item *)spec->handle; MLX5_ASSERT(mlx5_flex_index(dev->data->dev_private, tp) >= 0); for (i = 0; i < tp->mapnum; i++) { struct mlx5_flex_pattern_field *map = tp->map + i; uint32_t id = map->reg_id; uint32_t def = (RTE_BIT64(map->width) - 1) << map->shift; uint32_t val, msk; /* Skip placeholders for DUMMY fields. */ if (id == MLX5_INVALID_SAMPLE_REG_ID) { pos += map->width; continue; } val = mlx5_flex_get_bitfield(spec, pos, map->width, map->shift); msk = mlx5_flex_get_bitfield(mask, pos, map->width, map->shift); MLX5_ASSERT(map->width); MLX5_ASSERT(id < tp->devx_fp->num_samples); if (tp->tunnel_mode == FLEX_TUNNEL_MODE_MULTI && is_inner) { uint32_t num_samples = tp->devx_fp->num_samples / 2; MLX5_ASSERT(tp->devx_fp->num_samples % 2 == 0); MLX5_ASSERT(id < num_samples); id += num_samples; } mlx5_flex_set_match_sample(misc4_m, misc4_v, def, msk & def, val & msk & def, tp->devx_fp->sample_ids[id], id); pos += map->width; } } /** * Convert flex item handle (from the RTE flow) to flex item index on port. * Optionally can increment flex item object reference count. * * @param dev * Ethernet device to acquire flex item on. * @param[in] handle * Flow item handle from item spec. * @param[in] acquire * If set - increment reference counter. * * @return * >=0 - index on success, a negative errno value otherwise * and rte_errno is set. */ int mlx5_flex_acquire_index(struct rte_eth_dev *dev, struct rte_flow_item_flex_handle *handle, bool acquire) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flex_item *flex = (struct mlx5_flex_item *)handle; int ret = mlx5_flex_index(priv, flex); if (ret < 0) { errno = -EINVAL; rte_errno = EINVAL; return ret; } if (acquire) __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE); return ret; } /** * Release flex item index on port - decrements reference counter by index. * * @param dev * Ethernet device to acquire flex item on. * @param[in] index * Flow item index. * * @return * 0 - on success, a negative errno value otherwise and rte_errno is set. */ int mlx5_flex_release_index(struct rte_eth_dev *dev, int index) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flex_item *flex; if (index >= MLX5_PORT_FLEX_ITEM_NUM || !(priv->flex_item_map & (1u << index))) { errno = EINVAL; rte_errno = -EINVAL; return -EINVAL; } flex = priv->flex_item + index; if (flex->refcnt <= 1) { MLX5_ASSERT(false); errno = EINVAL; rte_errno = -EINVAL; return -EINVAL; } __atomic_sub_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE); return 0; } /* * Calculate largest mask value for a given shift. * * shift mask * ------- --------------- * 0 b111100 0x3C * 1 b111110 0x3E * 2 b111111 0x3F * 3 b011111 0x1F * 4 b001111 0x0F * 5 b000111 0x07 */ static uint8_t mlx5_flex_hdr_len_mask(uint8_t shift, const struct mlx5_hca_flex_attr *attr) { uint32_t base_mask; int diff = shift - MLX5_PARSE_GRAPH_NODE_HDR_LEN_SHIFT_DWORD; base_mask = mlx5_hca_parse_graph_node_base_hdr_len_mask(attr); return diff == 0 ? base_mask : diff < 0 ? (base_mask << -diff) & base_mask : base_mask >> diff; } static int mlx5_flex_translate_length(struct mlx5_hca_flex_attr *attr, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *devx, struct rte_flow_error *error) { const struct rte_flow_item_flex_field *field = &conf->next_header; struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; uint32_t len_width, mask; if (field->field_base % CHAR_BIT) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "not byte aligned header length field"); switch (field->field_mode) { case FIELD_MODE_DUMMY: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid header length field mode (DUMMY)"); case FIELD_MODE_FIXED: if (!(attr->header_length_mode & RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIXED))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported header length field mode (FIXED)"); if (field->field_size || field->offset_mask || field->offset_shift) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid fields for fixed mode"); if (field->field_base < 0) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "negative header length field base (FIXED)"); node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIXED; break; case FIELD_MODE_OFFSET: if (!(attr->header_length_mode & RTE_BIT32(MLX5_GRAPH_NODE_LEN_FIELD))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported header length field mode (OFFSET)"); node->header_length_mode = MLX5_GRAPH_NODE_LEN_FIELD; if (field->offset_mask == 0 || !rte_is_power_of_2(field->offset_mask + 1)) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid length field offset mask (OFFSET)"); len_width = rte_fls_u32(field->offset_mask); if (len_width > attr->header_length_mask_width) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "length field offset mask too wide (OFFSET)"); mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr); if (mask < field->offset_mask) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "length field shift too big (OFFSET)"); node->header_length_field_mask = RTE_MIN(mask, field->offset_mask); break; case FIELD_MODE_BITMASK: if (!(attr->header_length_mode & RTE_BIT32(MLX5_GRAPH_NODE_LEN_BITMASK))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported header length field mode (BITMASK)"); if (attr->header_length_mask_width < field->field_size) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "header length field width exceeds limit"); node->header_length_mode = MLX5_GRAPH_NODE_LEN_BITMASK; mask = mlx5_flex_hdr_len_mask(field->offset_shift, attr); if (mask < field->offset_mask) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "length field shift too big (BITMASK)"); node->header_length_field_mask = RTE_MIN(mask, field->offset_mask); break; default: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unknown header length field mode"); } if (field->field_base / CHAR_BIT >= 0 && field->field_base / CHAR_BIT > attr->max_base_header_length) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "header length field base exceeds limit"); node->header_length_base_value = field->field_base / CHAR_BIT; if (field->field_mode == FIELD_MODE_OFFSET || field->field_mode == FIELD_MODE_BITMASK) { if (field->offset_shift > 15 || field->offset_shift < 0) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "header length field shift exceeds limit"); node->header_length_field_shift = field->offset_shift; node->header_length_field_offset = field->offset_base; } return 0; } static int mlx5_flex_translate_next(struct mlx5_hca_flex_attr *attr, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *devx, struct rte_flow_error *error) { const struct rte_flow_item_flex_field *field = &conf->next_protocol; struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; switch (field->field_mode) { case FIELD_MODE_DUMMY: if (conf->nb_outputs) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "next protocol field is required (DUMMY)"); return 0; case FIELD_MODE_FIXED: break; case FIELD_MODE_OFFSET: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported next protocol field mode (OFFSET)"); break; case FIELD_MODE_BITMASK: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported next protocol field mode (BITMASK)"); default: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unknown next protocol field mode"); } MLX5_ASSERT(field->field_mode == FIELD_MODE_FIXED); if (!conf->nb_outputs) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "out link(s) is required if next field present"); if (attr->max_next_header_offset < field->field_base) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "next protocol field base exceeds limit"); if (field->offset_shift) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported next protocol field shift"); node->next_header_field_offset = field->field_base; node->next_header_field_size = field->field_size; return 0; } /* Helper structure to handle field bit intervals. */ struct mlx5_flex_field_cover { uint16_t num; int32_t start[MLX5_FLEX_ITEM_MAPPING_NUM]; int32_t end[MLX5_FLEX_ITEM_MAPPING_NUM]; uint8_t mapped[MLX5_FLEX_ITEM_MAPPING_NUM / CHAR_BIT + 1]; }; static void mlx5_flex_insert_field(struct mlx5_flex_field_cover *cover, uint16_t num, int32_t start, int32_t end) { MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM); MLX5_ASSERT(num <= cover->num); if (num < cover->num) { memmove(&cover->start[num + 1], &cover->start[num], (cover->num - num) * sizeof(int32_t)); memmove(&cover->end[num + 1], &cover->end[num], (cover->num - num) * sizeof(int32_t)); } cover->start[num] = start; cover->end[num] = end; cover->num++; } static void mlx5_flex_merge_field(struct mlx5_flex_field_cover *cover, uint16_t num) { uint32_t i, del = 0; int32_t end; MLX5_ASSERT(num < MLX5_FLEX_ITEM_MAPPING_NUM); MLX5_ASSERT(num < (cover->num - 1)); end = cover->end[num]; for (i = num + 1; i < cover->num; i++) { if (end < cover->start[i]) break; del++; if (end <= cover->end[i]) { cover->end[num] = cover->end[i]; break; } } if (del) { MLX5_ASSERT(del < (cover->num - 1u - num)); cover->num -= del; MLX5_ASSERT(cover->num > num); if ((cover->num - num) > 1) { memmove(&cover->start[num + 1], &cover->start[num + 1 + del], (cover->num - num - 1) * sizeof(int32_t)); memmove(&cover->end[num + 1], &cover->end[num + 1 + del], (cover->num - num - 1) * sizeof(int32_t)); } } } /* * Validate the sample field and update interval array * if parameters match with the 'match" field. * Returns: * < 0 - error * == 0 - no match, interval array not updated * > 0 - match, interval array updated */ static int mlx5_flex_cover_sample(struct mlx5_flex_field_cover *cover, struct rte_flow_item_flex_field *field, struct rte_flow_item_flex_field *match, struct mlx5_hca_flex_attr *attr, struct rte_flow_error *error) { int32_t start, end; uint32_t i; switch (field->field_mode) { case FIELD_MODE_DUMMY: return 0; case FIELD_MODE_FIXED: if (!(attr->sample_offset_mode & RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIXED))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported sample field mode (FIXED)"); if (field->offset_shift) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid sample field shift (FIXED"); if (field->field_base < 0) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid sample field base (FIXED)"); if (field->field_base / CHAR_BIT > attr->max_sample_base_offset) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "sample field base exceeds limit (FIXED)"); break; case FIELD_MODE_OFFSET: if (!(attr->sample_offset_mode & RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_FIELD))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported sample field mode (OFFSET)"); if (field->field_base / CHAR_BIT >= 0 && field->field_base / CHAR_BIT > attr->max_sample_base_offset) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "sample field base exceeds limit"); break; case FIELD_MODE_BITMASK: if (!(attr->sample_offset_mode & RTE_BIT32(MLX5_GRAPH_SAMPLE_OFFSET_BITMASK))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported sample field mode (BITMASK)"); if (field->field_base / CHAR_BIT >= 0 && field->field_base / CHAR_BIT > attr->max_sample_base_offset) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "sample field base exceeds limit"); break; default: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unknown data sample field mode"); } if (!match) { if (!field->field_size) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "zero sample field width"); if (field->field_id) DRV_LOG(DEBUG, "sample field id hint ignored"); } else { if (field->field_mode != match->field_mode || field->offset_base | match->offset_base || field->offset_mask | match->offset_mask || field->offset_shift | match->offset_shift) return 0; } start = field->field_base; end = start + field->field_size; /* Add the new or similar field to interval array. */ if (!cover->num) { cover->start[cover->num] = start; cover->end[cover->num] = end; cover->num = 1; return 1; } for (i = 0; i < cover->num; i++) { if (start > cover->end[i]) { if (i >= (cover->num - 1u)) { mlx5_flex_insert_field(cover, cover->num, start, end); break; } continue; } if (end < cover->start[i]) { mlx5_flex_insert_field(cover, i, start, end); break; } if (start < cover->start[i]) cover->start[i] = start; if (end > cover->end[i]) { cover->end[i] = end; if (i < (cover->num - 1u)) mlx5_flex_merge_field(cover, i); } break; } return 1; } static void mlx5_flex_config_sample(struct mlx5_devx_match_sample_attr *na, struct rte_flow_item_flex_field *field, enum rte_flow_item_flex_tunnel_mode tunnel_mode) { memset(na, 0, sizeof(struct mlx5_devx_match_sample_attr)); na->flow_match_sample_en = 1; switch (field->field_mode) { case FIELD_MODE_FIXED: na->flow_match_sample_offset_mode = MLX5_GRAPH_SAMPLE_OFFSET_FIXED; break; case FIELD_MODE_OFFSET: na->flow_match_sample_offset_mode = MLX5_GRAPH_SAMPLE_OFFSET_FIELD; na->flow_match_sample_field_offset = field->offset_base; na->flow_match_sample_field_offset_mask = field->offset_mask; na->flow_match_sample_field_offset_shift = field->offset_shift; break; case FIELD_MODE_BITMASK: na->flow_match_sample_offset_mode = MLX5_GRAPH_SAMPLE_OFFSET_BITMASK; na->flow_match_sample_field_offset = field->offset_base; na->flow_match_sample_field_offset_mask = field->offset_mask; na->flow_match_sample_field_offset_shift = field->offset_shift; break; default: MLX5_ASSERT(false); break; } switch (tunnel_mode) { case FLEX_TUNNEL_MODE_SINGLE: /* Fallthrough */ case FLEX_TUNNEL_MODE_TUNNEL: na->flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_FIRST; break; case FLEX_TUNNEL_MODE_MULTI: /* Fallthrough */ case FLEX_TUNNEL_MODE_OUTER: na->flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_OUTER; break; case FLEX_TUNNEL_MODE_INNER: na->flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_INNER; break; default: MLX5_ASSERT(false); break; } } /* Map specified field to set/subset of allocated sample registers. */ static int mlx5_flex_map_sample(struct rte_flow_item_flex_field *field, struct mlx5_flex_parser_devx *parser, struct mlx5_flex_item *item, struct rte_flow_error *error) { struct mlx5_devx_match_sample_attr node; int32_t start = field->field_base; int32_t end = start + field->field_size; struct mlx5_flex_pattern_field *trans; uint32_t i, done_bits = 0; if (field->field_mode == FIELD_MODE_DUMMY) { done_bits = field->field_size; while (done_bits) { uint32_t part = RTE_MIN(done_bits, sizeof(uint32_t) * CHAR_BIT); if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "too many flex item pattern translations"); trans = &item->map[item->mapnum]; trans->reg_id = MLX5_INVALID_SAMPLE_REG_ID; trans->shift = 0; trans->width = part; item->mapnum++; done_bits -= part; } return 0; } mlx5_flex_config_sample(&node, field, item->tunnel_mode); for (i = 0; i < parser->num_samples; i++) { struct mlx5_devx_match_sample_attr *sample = &parser->devx_conf.sample[i]; int32_t reg_start, reg_end; int32_t cov_start, cov_end; MLX5_ASSERT(sample->flow_match_sample_en); if (!sample->flow_match_sample_en) break; node.flow_match_sample_field_base_offset = sample->flow_match_sample_field_base_offset; if (memcmp(&node, sample, sizeof(node))) continue; reg_start = (int8_t)sample->flow_match_sample_field_base_offset; reg_start *= CHAR_BIT; reg_end = reg_start + 32; if (end <= reg_start || start >= reg_end) continue; cov_start = RTE_MAX(reg_start, start); cov_end = RTE_MIN(reg_end, end); MLX5_ASSERT(cov_end > cov_start); done_bits += cov_end - cov_start; if (item->mapnum >= MLX5_FLEX_ITEM_MAPPING_NUM) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "too many flex item pattern translations"); trans = &item->map[item->mapnum]; item->mapnum++; trans->reg_id = i; trans->shift = cov_start - reg_start; trans->width = cov_end - cov_start; } if (done_bits != field->field_size) { MLX5_ASSERT(false); return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "failed to map field to sample register"); } return 0; } /* Allocate sample registers for the specified field type and interval array. */ static int mlx5_flex_alloc_sample(struct mlx5_flex_field_cover *cover, struct mlx5_flex_parser_devx *parser, struct mlx5_flex_item *item, struct rte_flow_item_flex_field *field, struct mlx5_hca_flex_attr *attr, struct rte_flow_error *error) { struct mlx5_devx_match_sample_attr node; uint32_t idx = 0; mlx5_flex_config_sample(&node, field, item->tunnel_mode); while (idx < cover->num) { int32_t start, end; /* * Sample base offsets are in bytes, should be aligned * to 32-bit as required by firmware for samples. */ start = RTE_ALIGN_FLOOR(cover->start[idx], sizeof(uint32_t) * CHAR_BIT); node.flow_match_sample_field_base_offset = (start / CHAR_BIT) & 0xFF; /* Allocate sample register. */ if (parser->num_samples >= MLX5_GRAPH_NODE_SAMPLE_NUM || parser->num_samples >= attr->max_num_sample || parser->num_samples >= attr->max_num_prog_sample) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "no sample registers to handle all flex item fields"); parser->devx_conf.sample[parser->num_samples] = node; parser->num_samples++; /* Remove or update covered intervals. */ end = start + 32; while (idx < cover->num) { if (end >= cover->end[idx]) { idx++; continue; } if (end > cover->start[idx]) cover->start[idx] = end; break; } } return 0; } static int mlx5_flex_translate_sample(struct mlx5_hca_flex_attr *attr, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *parser, struct mlx5_flex_item *item, struct rte_flow_error *error) { struct mlx5_flex_field_cover cover; uint32_t i, j; int ret; switch (conf->tunnel) { case FLEX_TUNNEL_MODE_SINGLE: /* Fallthrough */ case FLEX_TUNNEL_MODE_OUTER: /* Fallthrough */ case FLEX_TUNNEL_MODE_INNER: /* Fallthrough */ case FLEX_TUNNEL_MODE_MULTI: /* Fallthrough */ case FLEX_TUNNEL_MODE_TUNNEL: break; default: return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unrecognized tunnel mode"); } item->tunnel_mode = conf->tunnel; if (conf->nb_samples > MLX5_FLEX_ITEM_MAPPING_NUM) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "sample field number exceeds limit"); /* * The application can specify fields smaller or bigger than 32 bits * covered with single sample register and it can specify field * offsets in any order. * * Gather all similar fields together, build array of bit intervals * in ascending order and try to cover with the smallest set of sample * registers. */ memset(&cover, 0, sizeof(cover)); for (i = 0; i < conf->nb_samples; i++) { struct rte_flow_item_flex_field *fl = conf->sample_data + i; /* Check whether field was covered in the previous iteration. */ if (cover.mapped[i / CHAR_BIT] & (1u << (i % CHAR_BIT))) continue; if (fl->field_mode == FIELD_MODE_DUMMY) continue; /* Build an interval array for the field and similar ones */ cover.num = 0; /* Add the first field to array unconditionally. */ ret = mlx5_flex_cover_sample(&cover, fl, NULL, attr, error); if (ret < 0) return ret; MLX5_ASSERT(ret > 0); cover.mapped[i / CHAR_BIT] |= 1u << (i % CHAR_BIT); for (j = i + 1; j < conf->nb_samples; j++) { struct rte_flow_item_flex_field *ft; /* Add field to array if its type matches. */ ft = conf->sample_data + j; ret = mlx5_flex_cover_sample(&cover, ft, fl, attr, error); if (ret < 0) return ret; if (!ret) continue; cover.mapped[j / CHAR_BIT] |= 1u << (j % CHAR_BIT); } /* Allocate sample registers to cover array of intervals. */ ret = mlx5_flex_alloc_sample(&cover, parser, item, fl, attr, error); if (ret) return ret; } /* Build the item pattern translating data on flow creation. */ item->mapnum = 0; memset(&item->map, 0, sizeof(item->map)); for (i = 0; i < conf->nb_samples; i++) { struct rte_flow_item_flex_field *fl = conf->sample_data + i; ret = mlx5_flex_map_sample(fl, parser, item, error); if (ret) { MLX5_ASSERT(false); return ret; } } if (conf->tunnel == FLEX_TUNNEL_MODE_MULTI) { /* * In FLEX_TUNNEL_MODE_MULTI tunnel mode PMD creates 2 sets * of samples. The first set is for outer and the second set * for inner flex flow item. Outer and inner samples differ * only in tunnel_mode. */ if (parser->num_samples > MLX5_GRAPH_NODE_SAMPLE_NUM / 2) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "no sample registers for inner"); rte_memcpy(parser->devx_conf.sample + parser->num_samples, parser->devx_conf.sample, parser->num_samples * sizeof(parser->devx_conf.sample[0])); for (i = 0; i < parser->num_samples; i++) { struct mlx5_devx_match_sample_attr *sm = i + parser->devx_conf.sample + parser->num_samples; sm->flow_match_sample_tunnel_mode = MLX5_GRAPH_SAMPLE_TUNNEL_INNER; } parser->num_samples *= 2; } return 0; } static int mlx5_flex_arc_type(enum rte_flow_item_type type, int in) { switch (type) { case RTE_FLOW_ITEM_TYPE_ETH: return MLX5_GRAPH_ARC_NODE_MAC; case RTE_FLOW_ITEM_TYPE_IPV4: return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV4; case RTE_FLOW_ITEM_TYPE_IPV6: return in ? MLX5_GRAPH_ARC_NODE_IP : MLX5_GRAPH_ARC_NODE_IPV6; case RTE_FLOW_ITEM_TYPE_UDP: return MLX5_GRAPH_ARC_NODE_UDP; case RTE_FLOW_ITEM_TYPE_TCP: return MLX5_GRAPH_ARC_NODE_TCP; case RTE_FLOW_ITEM_TYPE_MPLS: return MLX5_GRAPH_ARC_NODE_MPLS; case RTE_FLOW_ITEM_TYPE_GRE: return MLX5_GRAPH_ARC_NODE_GRE; case RTE_FLOW_ITEM_TYPE_GENEVE: return MLX5_GRAPH_ARC_NODE_GENEVE; case RTE_FLOW_ITEM_TYPE_VXLAN_GPE: return MLX5_GRAPH_ARC_NODE_VXLAN_GPE; default: return -EINVAL; } } static int mlx5_flex_arc_in_eth(const struct rte_flow_item *item, struct rte_flow_error *error) { const struct rte_flow_item_eth *spec = item->spec; const struct rte_flow_item_eth *mask = item->mask; struct rte_flow_item_eth eth = { .hdr.ether_type = RTE_BE16(0xFFFF) }; if (memcmp(mask, ð, sizeof(struct rte_flow_item_eth))) { return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "invalid eth item mask"); } return rte_be_to_cpu_16(spec->hdr.ether_type); } static int mlx5_flex_arc_in_udp(const struct rte_flow_item *item, struct rte_flow_error *error) { const struct rte_flow_item_udp *spec = item->spec; const struct rte_flow_item_udp *mask = item->mask; struct rte_flow_item_udp udp = { .hdr.dst_port = RTE_BE16(0xFFFF) }; if (memcmp(mask, &udp, sizeof(struct rte_flow_item_udp))) { return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, item, "invalid eth item mask"); } return rte_be_to_cpu_16(spec->hdr.dst_port); } static int mlx5_flex_translate_arc_in(struct mlx5_hca_flex_attr *attr, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *devx, struct mlx5_flex_item *item, struct rte_flow_error *error) { struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; uint32_t i; RTE_SET_USED(item); if (conf->nb_inputs > attr->max_num_arc_in) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "too many input links"); for (i = 0; i < conf->nb_inputs; i++) { struct mlx5_devx_graph_arc_attr *arc = node->in + i; struct rte_flow_item_flex_link *link = conf->input_link + i; const struct rte_flow_item *rte_item = &link->item; int arc_type; int ret; if (!rte_item->spec || !rte_item->mask || rte_item->last) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid flex item IN arc format"); arc_type = mlx5_flex_arc_type(rte_item->type, true); if (arc_type < 0 || !(attr->node_in & RTE_BIT32(arc_type))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported flex item IN arc type"); arc->arc_parse_graph_node = arc_type; arc->start_inner_tunnel = 0; /* * Configure arc IN condition value. The value location depends * on protocol. Current FW version supports IP & UDP for IN * arcs only, and locations for these protocols are defined. * Add more protocols when available. */ switch (rte_item->type) { case RTE_FLOW_ITEM_TYPE_ETH: ret = mlx5_flex_arc_in_eth(rte_item, error); break; case RTE_FLOW_ITEM_TYPE_UDP: ret = mlx5_flex_arc_in_udp(rte_item, error); break; default: MLX5_ASSERT(false); return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported flex item IN arc type"); } if (ret < 0) return ret; arc->compare_condition_value = (uint16_t)ret; } return 0; } static int mlx5_flex_translate_arc_out(struct mlx5_hca_flex_attr *attr, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *devx, struct mlx5_flex_item *item, struct rte_flow_error *error) { struct mlx5_devx_graph_node_attr *node = &devx->devx_conf; bool is_tunnel = conf->tunnel == FLEX_TUNNEL_MODE_TUNNEL; uint32_t i; RTE_SET_USED(item); if (conf->nb_outputs > attr->max_num_arc_out) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "too many output links"); for (i = 0; i < conf->nb_outputs; i++) { struct mlx5_devx_graph_arc_attr *arc = node->out + i; struct rte_flow_item_flex_link *link = conf->output_link + i; const struct rte_flow_item *rte_item = &link->item; int arc_type; if (rte_item->spec || rte_item->mask || rte_item->last) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "flex node: invalid OUT arc format"); arc_type = mlx5_flex_arc_type(rte_item->type, false); if (arc_type < 0 || !(attr->node_out & RTE_BIT32(arc_type))) return rte_flow_error_set (error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "unsupported flex item OUT arc type"); arc->arc_parse_graph_node = arc_type; arc->start_inner_tunnel = !!is_tunnel; arc->compare_condition_value = link->next; } return 0; } /* Translate RTE flex item API configuration into flaex parser settings. */ static int mlx5_flex_translate_conf(struct rte_eth_dev *dev, const struct rte_flow_item_flex_conf *conf, struct mlx5_flex_parser_devx *devx, struct mlx5_flex_item *item, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_hca_flex_attr *attr = &priv->sh->cdev->config.hca_attr.flex; int ret; ret = mlx5_flex_translate_length(attr, conf, devx, error); if (ret) return ret; ret = mlx5_flex_translate_next(attr, conf, devx, error); if (ret) return ret; ret = mlx5_flex_translate_sample(attr, conf, devx, item, error); if (ret) return ret; ret = mlx5_flex_translate_arc_in(attr, conf, devx, item, error); if (ret) return ret; ret = mlx5_flex_translate_arc_out(attr, conf, devx, item, error); if (ret) return ret; return 0; } /** * Create the flex item with specified configuration over the Ethernet device. * * @param dev * Ethernet device to create flex item on. * @param[in] conf * Flex item configuration. * @param[out] error * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. * * @return * Non-NULL opaque pointer on success, NULL otherwise and rte_errno is set. */ struct rte_flow_item_flex_handle * flow_dv_item_create(struct rte_eth_dev *dev, const struct rte_flow_item_flex_conf *conf, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flex_parser_devx devx_config = { .devx_obj = NULL }; struct mlx5_flex_item *flex; struct mlx5_list_entry *ent; MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); flex = mlx5_flex_alloc(priv); if (!flex) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "too many flex items created on the port"); return NULL; } if (mlx5_flex_translate_conf(dev, conf, &devx_config, flex, error)) goto error; ent = mlx5_list_register(priv->sh->flex_parsers_dv, &devx_config); if (!ent) { rte_flow_error_set(error, ENOMEM, RTE_FLOW_ERROR_TYPE_UNSPECIFIED, NULL, "flex item creation failure"); goto error; } flex->devx_fp = container_of(ent, struct mlx5_flex_parser_devx, entry); /* Mark initialized flex item valid. */ __atomic_add_fetch(&flex->refcnt, 1, __ATOMIC_RELEASE); return (struct rte_flow_item_flex_handle *)flex; error: mlx5_flex_free(priv, flex); return NULL; } /** * Release the flex item on the specified Ethernet device. * * @param dev * Ethernet device to destroy flex item on. * @param[in] handle * Handle of the item existing on the specified device. * @param[out] error * Perform verbose error reporting if not NULL. PMDs initialize this * structure in case of error only. * * @return * 0 on success, a negative errno value otherwise and rte_errno is set. */ int flow_dv_item_release(struct rte_eth_dev *dev, const struct rte_flow_item_flex_handle *handle, struct rte_flow_error *error) { struct mlx5_priv *priv = dev->data->dev_private; struct mlx5_flex_item *flex = (struct mlx5_flex_item *)(uintptr_t)handle; uint32_t old_refcnt = 1; int rc; MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY); rte_spinlock_lock(&priv->flex_item_sl); if (mlx5_flex_index(priv, flex) < 0) { rte_spinlock_unlock(&priv->flex_item_sl); return rte_flow_error_set(error, EINVAL, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "invalid flex item handle value"); } if (!__atomic_compare_exchange_n(&flex->refcnt, &old_refcnt, 0, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) { rte_spinlock_unlock(&priv->flex_item_sl); return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "flex item has flow references"); } /* Flex item is marked as invalid, we can leave locked section. */ rte_spinlock_unlock(&priv->flex_item_sl); MLX5_ASSERT(flex->devx_fp); rc = mlx5_list_unregister(priv->sh->flex_parsers_dv, &flex->devx_fp->entry); flex->devx_fp = NULL; mlx5_flex_free(priv, flex); if (rc < 0) return rte_flow_error_set(error, EBUSY, RTE_FLOW_ERROR_TYPE_ITEM, NULL, "flex item release failure"); return 0; } /* DevX flex parser list callbacks. */ struct mlx5_list_entry * mlx5_flex_parser_create_cb(void *list_ctx, void *ctx) { struct mlx5_dev_ctx_shared *sh = list_ctx; struct mlx5_flex_parser_devx *fp, *conf = ctx; int ret; fp = mlx5_malloc(MLX5_MEM_ZERO, sizeof(struct mlx5_flex_parser_devx), 0, SOCKET_ID_ANY); if (!fp) return NULL; /* Copy the requested configurations. */ fp->num_samples = conf->num_samples; memcpy(&fp->devx_conf, &conf->devx_conf, sizeof(fp->devx_conf)); /* Create DevX flex parser. */ fp->devx_obj = mlx5_devx_cmd_create_flex_parser(sh->cdev->ctx, &fp->devx_conf); if (!fp->devx_obj) goto error; /* Query the firmware assigned sample ids. */ ret = mlx5_devx_cmd_query_parse_samples(fp->devx_obj, fp->sample_ids, fp->num_samples); if (ret) goto error; DRV_LOG(DEBUG, "DEVx flex parser %p created, samples num: %u", (const void *)fp, fp->num_samples); return &fp->entry; error: if (fp->devx_obj) mlx5_devx_cmd_destroy((void *)(uintptr_t)fp->devx_obj); if (fp) mlx5_free(fp); return NULL; } int mlx5_flex_parser_match_cb(void *list_ctx, struct mlx5_list_entry *iter, void *ctx) { struct mlx5_flex_parser_devx *fp = container_of(iter, struct mlx5_flex_parser_devx, entry); struct mlx5_flex_parser_devx *org = container_of(ctx, struct mlx5_flex_parser_devx, entry); RTE_SET_USED(list_ctx); return !iter || !ctx || memcmp(&fp->devx_conf, &org->devx_conf, sizeof(fp->devx_conf)); } void mlx5_flex_parser_remove_cb(void *list_ctx, struct mlx5_list_entry *entry) { struct mlx5_flex_parser_devx *fp = container_of(entry, struct mlx5_flex_parser_devx, entry); RTE_SET_USED(list_ctx); MLX5_ASSERT(fp->devx_obj); claim_zero(mlx5_devx_cmd_destroy(fp->devx_obj)); DRV_LOG(DEBUG, "DEVx flex parser %p destroyed", (const void *)fp); mlx5_free(entry); } struct mlx5_list_entry * mlx5_flex_parser_clone_cb(void *list_ctx, struct mlx5_list_entry *entry, void *ctx) { struct mlx5_flex_parser_devx *fp; RTE_SET_USED(list_ctx); RTE_SET_USED(entry); fp = mlx5_malloc(0, sizeof(struct mlx5_flex_parser_devx), 0, SOCKET_ID_ANY); if (!fp) return NULL; memcpy(fp, ctx, sizeof(struct mlx5_flex_parser_devx)); return &fp->entry; } void mlx5_flex_parser_clone_free_cb(void *list_ctx, struct mlx5_list_entry *entry) { struct mlx5_flex_parser_devx *fp = container_of(entry, struct mlx5_flex_parser_devx, entry); RTE_SET_USED(list_ctx); mlx5_free(fp); }