From ce7a3c52835809c106cd49b2ac6fba6b808fda50 Mon Sep 17 00:00:00 2001 From: slavash Date: Wed, 5 Dec 2018 14:21:28 +0000 Subject: [PATCH] mlx5en: Fix for inlining issues in transmit path 1) Don't exceed the drivers own hardcoded TX inline limit. The blueflame register size can be much greater than the hardcoded limit for inlining. Make sure we don't exceed the drivers own limit, because this also means that the maximum number of TX fragments becomes invalid and then memory size assumptions in the TX path no longer hold up. 2) Make sure the mlx5_query_min_inline() function returns an error code. 3) Header inlining is required when using TSO. 4) Catch failure to compute inline header size for TSO. 5) Add support for UDP when computing inline header size. 6) Fix for inlining issues with regards to DSCP. Make sure we inline 4 bytes beyond the ethernet and/or VLAN header to workaround a hardware bug extracting the DSCP field from the IPv4/v6 header. Submitted by: hselasky@ Approved by: hselasky (mentor) MFC after: 1 week Sponsored by: Mellanox Technologies --- sys/dev/mlx5/mlx5_core/mlx5_vport.c | 14 +- sys/dev/mlx5/mlx5_en/en.h | 7 +- sys/dev/mlx5/mlx5_en/en_rl.h | 1 + sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c | 6 + sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 95 ++++++++--- sys/dev/mlx5/mlx5_en/mlx5_en_rl.c | 31 +++- sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 219 ++++++++++++++++--------- sys/dev/mlx5/vport.h | 2 +- 8 files changed, 260 insertions(+), 115 deletions(-) diff --git a/sys/dev/mlx5/mlx5_core/mlx5_vport.c b/sys/dev/mlx5/mlx5_core/mlx5_vport.c index 431872f477d5..f9e70c8939f4 100644 --- a/sys/dev/mlx5/mlx5_core/mlx5_vport.c +++ b/sys/dev/mlx5/mlx5_core/mlx5_vport.c @@ -222,20 +222,28 @@ int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev, } EXPORT_SYMBOL_GPL(mlx5_query_nic_vport_min_inline); -void mlx5_query_min_inline(struct mlx5_core_dev *mdev, - u8 *min_inline_mode) +int mlx5_query_min_inline(struct mlx5_core_dev *mdev, + u8 *min_inline_mode) { + int err; + switch (MLX5_CAP_ETH(mdev, wqe_inline_mode)) { case MLX5_CAP_INLINE_MODE_L2: *min_inline_mode = MLX5_INLINE_MODE_L2; + err = 0; break; case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: - mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode); + err = mlx5_query_nic_vport_min_inline(mdev, 0, min_inline_mode); break; case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: *min_inline_mode = MLX5_INLINE_MODE_NONE; + err = 0; + break; + default: + err = -EINVAL; break; } + return err; } EXPORT_SYMBOL_GPL(mlx5_query_min_inline); diff --git a/sys/dev/mlx5/mlx5_en/en.h b/sys/dev/mlx5/mlx5_en/en.h index 2a30390549da..83fa897408e4 100644 --- a/sys/dev/mlx5/mlx5_en/en.h +++ b/sys/dev/mlx5/mlx5_en/en.h @@ -619,7 +619,9 @@ struct mlx5e_sq { u32 mkey_be; u16 max_inline; u8 min_inline_mode; - u8 vlan_inline_cap; + u8 min_insert_caps; +#define MLX5E_INSERT_VLAN 1 +#define MLX5E_INSERT_NON_VLAN 2 /* control path */ struct mlx5_wq_ctrl wq_ctrl; @@ -925,6 +927,7 @@ void mlx5e_drain_sq(struct mlx5e_sq *); void mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value); void mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value); void mlx5e_resume_sq(struct mlx5e_sq *sq); -u8 mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev); +void mlx5e_update_sq_inline(struct mlx5e_sq *sq); +void mlx5e_refresh_sq_inline(struct mlx5e_priv *priv); #endif /* _MLX5_EN_H_ */ diff --git a/sys/dev/mlx5/mlx5_en/en_rl.h b/sys/dev/mlx5/mlx5_en/en_rl.h index 4e2c6c539857..dcbfedc46380 100644 --- a/sys/dev/mlx5/mlx5_en/en_rl.h +++ b/sys/dev/mlx5/mlx5_en/en_rl.h @@ -166,6 +166,7 @@ struct mlx5e_rl_priv_data { int mlx5e_rl_init(struct mlx5e_priv *priv); void mlx5e_rl_cleanup(struct mlx5e_priv *priv); +void mlx5e_rl_refresh_sq_inline(struct mlx5e_rl_priv_data *rl); if_snd_tag_alloc_t mlx5e_rl_snd_tag_alloc; if_snd_tag_modify_t mlx5e_rl_snd_tag_modify; if_snd_tag_query_t mlx5e_rl_snd_tag_query; diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c index 6e5370e62999..c3bd4f3f6aaa 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c @@ -374,6 +374,12 @@ mlx5e_trust_state_handler(SYSCTL_HANDLER_ARGS) goto done; priv->params_ethtool.trust_state = result; + + /* update inline mode */ + mlx5e_refresh_sq_inline(priv); +#ifdef RATELIMIT + mlx5e_rl_refresh_sq_inline(&priv->rl); +#endif done: PRIV_UNLOCK(priv); return (err); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c index fa15f31237e7..429bb0e3b481 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_main.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_main.c @@ -1126,6 +1126,52 @@ static const char *mlx5e_sq_stats_desc[] = { MLX5E_SQ_STATS(MLX5E_STATS_DESC) }; +void +mlx5e_update_sq_inline(struct mlx5e_sq *sq) +{ + sq->max_inline = sq->priv->params.tx_max_inline; + sq->min_inline_mode = sq->priv->params.tx_min_inline_mode; + + /* + * Check if trust state is DSCP or if inline mode is NONE which + * indicates CX-5 or newer hardware. + */ + if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP || + sq->min_inline_mode == MLX5_INLINE_MODE_NONE) { + if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert)) + sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN; + else + sq->min_insert_caps = MLX5E_INSERT_NON_VLAN; + } else { + sq->min_insert_caps = 0; + } +} + +static void +mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c) +{ + int i; + + for (i = 0; i != c->num_tc; i++) { + mtx_lock(&c->sq[i].lock); + mlx5e_update_sq_inline(&c->sq[i]); + mtx_unlock(&c->sq[i].lock); + } +} + +void +mlx5e_refresh_sq_inline(struct mlx5e_priv *priv) +{ + int i; + + /* check if channels are closed */ + if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0) + return; + + for (i = 0; i < priv->params.num_channels; i++) + mlx5e_refresh_sq_inline_sub(priv, priv->channel[i]); +} + static int mlx5e_create_sq(struct mlx5e_channel *c, int tc, @@ -1180,9 +1226,8 @@ mlx5e_create_sq(struct mlx5e_channel *c, sq->ifp = priv->ifp; sq->priv = priv; sq->tc = tc; - sq->max_inline = priv->params.tx_max_inline; - sq->min_inline_mode = priv->params.tx_min_inline_mode; - sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert); + + mlx5e_update_sq_inline(sq); snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc); mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet), @@ -2992,18 +3037,24 @@ mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) static u16 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev) { - int bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2; + uint32_t bf_buf_size = (1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U; - return bf_buf_size - - sizeof(struct mlx5e_tx_wqe) + - 2 /*sizeof(mlx5e_tx_wqe.inline_hdr_start)*/; + bf_buf_size -= sizeof(struct mlx5e_tx_wqe) - 2; + + /* verify against driver hardware limit */ + if (bf_buf_size > MLX5E_MAX_TX_INLINE) + bf_buf_size = MLX5E_MAX_TX_INLINE; + + return (bf_buf_size); } -static void +static int mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, struct mlx5e_priv *priv, int num_comp_vectors) { + int err; + /* * TODO: Consider link speed for setting "log_sq_size", * "log_rq_size" and "cq_moderation_xxx": @@ -3035,7 +3086,10 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, priv->params.default_vlan_prio = 0; priv->counter_set_id = -1; priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev); - mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode); + + err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode); + if (err) + return (err); /* * hw lro is currently defaulted to off. when it won't anymore we @@ -3058,6 +3112,8 @@ mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev, INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); + + return (0); } static int @@ -3297,20 +3353,6 @@ mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value) } } -u8 -mlx5e_params_calculate_tx_min_inline(struct mlx5_core_dev *mdev) -{ - u8 min_inline_mode; - - min_inline_mode = MLX5_INLINE_MODE_L2; - mlx5_query_min_inline(mdev, &min_inline_mode); - if (min_inline_mode == MLX5_INLINE_MODE_NONE && - !MLX5_CAP_ETH(mdev, wqe_vlan_insert)) - min_inline_mode = MLX5_INLINE_MODE_L2; - - return (min_inline_mode); -} - static void mlx5e_add_hw_stats(struct mlx5e_priv *priv) { @@ -3590,7 +3632,12 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev) mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n"); goto err_free_sysctl; } - mlx5e_build_ifp_priv(mdev, priv, ncv); + + err = mlx5e_build_ifp_priv(mdev, priv, ncv); + if (err) { + mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err); + goto err_free_sysctl; + } snprintf(unit, sizeof(unit), "mce%u_wq", device_get_unit(mdev->pdev->dev.bsddev)); diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c b/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c index cf0b32ec4b56..32f3f53f7b07 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_rl.c @@ -137,9 +137,8 @@ mlx5e_rl_create_sq(struct mlx5e_priv *priv, struct mlx5e_sq *sq, sq->mkey_be = cpu_to_be32(priv->mr.key); sq->ifp = priv->ifp; sq->priv = priv; - sq->max_inline = priv->params.tx_max_inline; - sq->min_inline_mode = priv->params.tx_min_inline_mode; - sq->vlan_inline_cap = MLX5_CAP_ETH(mdev, wqe_vlan_insert); + + mlx5e_update_sq_inline(sq); return (0); @@ -1233,6 +1232,32 @@ mlx5e_rl_refresh_channel_params(struct mlx5e_rl_priv_data *rl) return (0); } +void +mlx5e_rl_refresh_sq_inline(struct mlx5e_rl_priv_data *rl) +{ + uint64_t x; + uint64_t y; + + for (y = 0; y != rl->param.tx_worker_threads_def; y++) { + struct mlx5e_rl_worker *rlw = rl->workers + y; + + for (x = 0; x != rl->param.tx_channels_per_worker_def; x++) { + struct mlx5e_rl_channel *channel; + struct mlx5e_sq *sq; + + channel = rlw->channels + x; + sq = channel->sq; + + if (sq == NULL) + continue; + + mtx_lock(&sq->lock); + mlx5e_update_sq_inline(sq); + mtx_unlock(&sq->lock); + } + } +} + static int mlx5e_rl_tx_limit_add(struct mlx5e_rl_priv_data *rl, uint64_t value) { diff --git a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c index e7ee7f41c574..f8b58ed96aaa 100644 --- a/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c +++ b/sys/dev/mlx5/mlx5_en/mlx5_en_tx.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 Mellanox Technologies. All rights reserved. + * Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -154,49 +154,53 @@ mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) } static inline u16 -mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb) +mlx5e_get_l2_header_size(struct mlx5e_sq *sq, struct mbuf *mb) { + struct ether_vlan_header *eh; + uint16_t eth_type; + int min_inline; - switch(sq->min_inline_mode) { - case MLX5_INLINE_MODE_NONE: - /* - * When inline mode is NONE, we do not need to copy - * headers into WQEs, except when vlan tag framing is - * requested. Hardware might offload vlan tagging on - * transmit. This is a separate capability, which is - * known to be disabled on ConnectX-5 due to a hardware - * bug RM 931383. If vlan_inline_cap is not present and - * the packet has vlan tag, fall back to inlining. - */ - if ((mb->m_flags & M_VLANTAG) != 0 && - sq->vlan_inline_cap == 0) - break; - return (0); - case MLX5_INLINE_MODE_L2: - /* - * Due to hardware limitations, when trust mode is - * DSCP, the hardware may request MLX5_INLINE_MODE_L2 - * while it really needs all L2 headers and the 4 first - * bytes of the IP header (which include the - * TOS/traffic-class). - * - * To avoid doing a firmware command for querying the - * trust state and parsing the mbuf for doing - * unnecessary checks (VLAN/eth_type) in the fast path, - * we are going for the worth case (22 Bytes) if - * the mb->m_pkthdr.len allows it. - */ - if (mb->m_pkthdr.len > ETHER_HDR_LEN + - ETHER_VLAN_ENCAP_LEN + 4) - return (MIN(sq->max_inline, ETHER_HDR_LEN + - ETHER_VLAN_ENCAP_LEN + 4)); - break; + eh = mtod(mb, struct ether_vlan_header *); + if (unlikely(mb->m_len < ETHER_HDR_LEN)) { + goto max_inline; + } else if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + if (unlikely(mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN))) + goto max_inline; + eth_type = ntohs(eh->evl_proto); + min_inline = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; + } else { + eth_type = ntohs(eh->evl_encap_proto); + min_inline = ETHER_HDR_LEN; } - return (MIN(sq->max_inline, mb->m_pkthdr.len)); + + switch (eth_type) { + case ETHERTYPE_IP: + case ETHERTYPE_IPV6: + /* + * Make sure the TOS(IPv4) or traffic class(IPv6) + * field gets inlined. Else the SQ may stall. + */ + min_inline += 4; + break; + default: + goto max_inline; + } + + /* + * m_copydata() will be used on the remaining header which + * does not need to reside within the first m_len bytes of + * data: + */ + if (mb->m_pkthdr.len < min_inline) + goto max_inline; + return (min_inline); + +max_inline: + return (MIN(mb->m_pkthdr.len, sq->max_inline)); } static int -mlx5e_get_header_size(struct mbuf *mb) +mlx5e_get_full_header_size(struct mbuf *mb) { struct ether_vlan_header *eh; struct tcphdr *th; @@ -210,31 +214,46 @@ mlx5e_get_header_size(struct mbuf *mb) if (mb->m_len < ETHER_HDR_LEN) return (0); if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { + if (mb->m_len < (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN)) + return (0); eth_type = ntohs(eh->evl_proto); eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; } else { eth_type = ntohs(eh->evl_encap_proto); eth_hdr_len = ETHER_HDR_LEN; } - if (mb->m_len < eth_hdr_len) - return (0); switch (eth_type) { case ETHERTYPE_IP: ip = (struct ip *)(mb->m_data + eth_hdr_len); if (mb->m_len < eth_hdr_len + sizeof(*ip)) return (0); - if (ip->ip_p != IPPROTO_TCP) + switch (ip->ip_p) { + case IPPROTO_TCP: + ip_hlen = ip->ip_hl << 2; + eth_hdr_len += ip_hlen; + break; + case IPPROTO_UDP: + ip_hlen = ip->ip_hl << 2; + eth_hdr_len += ip_hlen + 8; + goto done; + default: return (0); - ip_hlen = ip->ip_hl << 2; - eth_hdr_len += ip_hlen; + } break; case ETHERTYPE_IPV6: ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); if (mb->m_len < eth_hdr_len + sizeof(*ip6)) return (0); - if (ip6->ip6_nxt != IPPROTO_TCP) + switch (ip6->ip6_nxt) { + case IPPROTO_TCP: + eth_hdr_len += sizeof(*ip6); + break; + case IPPROTO_UDP: + eth_hdr_len += sizeof(*ip6) + 8; + goto done; + default: return (0); - eth_hdr_len += sizeof(*ip6); + } break; default: return (0); @@ -244,7 +263,13 @@ mlx5e_get_header_size(struct mbuf *mb) th = (struct tcphdr *)(mb->m_data + eth_hdr_len); tcp_hlen = th->th_off << 2; eth_hdr_len += tcp_hlen; - if (mb->m_len < eth_hdr_len) +done: + /* + * m_copydata() will be used on the remaining header which + * does not need to reside within the first m_len bytes of + * data: + */ + if (mb->m_pkthdr.len < eth_hdr_len) return (0); return (eth_hdr_len); } @@ -306,7 +331,11 @@ mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) wqe->eth.mss = cpu_to_be16(mss); opcode = MLX5_OPCODE_LSO; - ihs = mlx5e_get_header_size(mb); + ihs = mlx5e_get_full_header_size(mb); + if (unlikely(ihs == 0)) { + err = EINVAL; + goto tx_drop; + } payload_len = mb->m_pkthdr.len - ihs; if (payload_len == 0) num_pkts = 1; @@ -318,46 +347,72 @@ mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) sq->stats.tso_bytes += payload_len; } else { opcode = MLX5_OPCODE_SEND; - ihs = mlx5e_get_inline_hdr_size(sq, mb); + + switch (sq->min_inline_mode) { + case MLX5_INLINE_MODE_IP: + case MLX5_INLINE_MODE_TCP_UDP: + ihs = mlx5e_get_full_header_size(mb); + if (unlikely(ihs == 0)) + ihs = mlx5e_get_l2_header_size(sq, mb); + break; + case MLX5_INLINE_MODE_L2: + ihs = mlx5e_get_l2_header_size(sq, mb); + break; + case MLX5_INLINE_MODE_NONE: + /* FALLTHROUGH */ + default: + if ((mb->m_flags & M_VLANTAG) != 0 && + (sq->min_insert_caps & MLX5E_INSERT_VLAN) != 0) { + /* inlining VLAN data is not required */ + wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */ + wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag); + ihs = 0; + } else if ((mb->m_flags & M_VLANTAG) == 0 && + (sq->min_insert_caps & MLX5E_INSERT_NON_VLAN) != 0) { + /* inlining non-VLAN data is not required */ + ihs = 0; + } else { + /* we are forced to inlining L2 header, if any */ + ihs = mlx5e_get_l2_header_size(sq, mb); + } + break; + } sq->mbuf[pi].num_bytes = max_t (unsigned int, mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); } - if (ihs == 0) { - if ((mb->m_flags & M_VLANTAG) != 0) { - wqe->eth.vlan_cmd = htons(0x8000); /* bit 0 CVLAN */ - wqe->eth.vlan_hdr = htons(mb->m_pkthdr.ether_vtag); - } else { - wqe->eth.inline_hdr_sz = 0; - } - } else { - if ((mb->m_flags & M_VLANTAG) != 0) { - struct ether_vlan_header *eh = (struct ether_vlan_header - *)wqe->eth.inline_hdr_start; - /* Range checks */ - if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) - ihs = (MLX5E_MAX_TX_INLINE - - ETHER_VLAN_ENCAP_LEN); - else if (ihs < ETHER_HDR_LEN) { - err = EINVAL; - goto tx_drop; - } - m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); - m_adj(mb, ETHER_HDR_LEN); - /* Insert 4 bytes VLAN tag into data stream */ - eh->evl_proto = eh->evl_encap_proto; - eh->evl_encap_proto = htons(ETHERTYPE_VLAN); - eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); - /* Copy rest of header data, if any */ - m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + - 1)); - m_adj(mb, ihs - ETHER_HDR_LEN); - /* Extend header by 4 bytes */ - ihs += ETHER_VLAN_ENCAP_LEN; - } else { - m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); - m_adj(mb, ihs); + if (likely(ihs == 0)) { + /* nothing to inline */ + } else if (unlikely(ihs > sq->max_inline)) { + /* inline header size is too big */ + err = EINVAL; + goto tx_drop; + } else if ((mb->m_flags & M_VLANTAG) != 0) { + struct ether_vlan_header *eh = (struct ether_vlan_header *) + wqe->eth.inline_hdr_start; + + /* Range checks */ + if (unlikely(ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN))) + ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN); + else if (unlikely(ihs < ETHER_HDR_LEN)) { + err = EINVAL; + goto tx_drop; } + m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); + m_adj(mb, ETHER_HDR_LEN); + /* Insert 4 bytes VLAN tag into data stream */ + eh->evl_proto = eh->evl_encap_proto; + eh->evl_encap_proto = htons(ETHERTYPE_VLAN); + eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); + /* Copy rest of header data, if any */ + m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1)); + m_adj(mb, ihs - ETHER_HDR_LEN); + /* Extend header by 4 bytes */ + ihs += ETHER_VLAN_ENCAP_LEN; + wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); + } else { + m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); + m_adj(mb, ihs); wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); } diff --git a/sys/dev/mlx5/vport.h b/sys/dev/mlx5/vport.h index 06aa8ed34897..61dc9c4880cf 100644 --- a/sys/dev/mlx5/vport.h +++ b/sys/dev/mlx5/vport.h @@ -88,7 +88,7 @@ int mlx5_set_nic_vport_current_mac(struct mlx5_core_dev *mdev, int vport, bool other_vport, u8 *addr); int mlx5_query_nic_vport_min_inline(struct mlx5_core_dev *mdev, u16 vport, u8 *min_inline); -void mlx5_query_min_inline(struct mlx5_core_dev *mdev, u8 *min_inline); +int mlx5_query_min_inline(struct mlx5_core_dev *mdev, u8 *min_inline); int mlx5_modify_nic_vport_min_inline(struct mlx5_core_dev *mdev, u16 vport, u8 min_inline); int mlx5_modify_nic_vport_port_guid(struct mlx5_core_dev *mdev,