d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mlx5en: Implement TLS RX support.

Browse Source 
TLS RX support is modeled after TLS TX support. The basic structures and layouts
are almost identical, except that the send tag created filters RX traffic and
not TX traffic.

The TLS RX tag keeps track of past TLS records up to a certain limit,
approximately 1 Gbyte of TCP data. TLS records of same length are joined
into a single database record.

Regularly the HW is queried for TLS RX progress information. The TCP sequence
number gotten from the HW is then matches against the database of TLS TCP
sequence number records and lengths. If a match is found a static params WQE
is queued on the IQ and the hardware should immediately resume decrypting TLS
data until the next non-sequential TCP packet arrives.

Offloading TLS RX data is supported for untagged, prio-tagged, and
regular VLAN traffic.

MFC after:	1 week
Sponsored by:	NVIDIA Networking
This commit is contained in:
Hans Petter Selasky 2022-02-01 16:20:16 +01:00
parent e6d7ac1d03
commit 84d7b8e75f
9 changed files with 1228 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
sys/conf/files
View File

@ -4929,6 +4929,8 @@ dev/mlx5/mlx5_en/mlx5_en_flow_table.c optional mlx5en pci inet inet6 \
compile-with "${OFED_C}"
dev/mlx5/mlx5_en/mlx5_en_hw_tls.c optional mlx5en pci inet inet6 \
compile-with "${OFED_C}"
dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c optional mlx5en pci inet inet6 \
compile-with "${OFED_C}"
dev/mlx5/mlx5_en/mlx5_en_iq.c optional mlx5en pci inet inet6 \
compile-with "${OFED_C}"
dev/mlx5/mlx5_en/mlx5_en_rx.c optional mlx5en pci inet inet6 \

20
sys/dev/mlx5/device.h
View File

@ -408,6 +408,14 @@ enum {
MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS = 0x2,
};
struct mlx5_wqe_tls_static_params_seg {
u8 ctx[MLX5_ST_SZ_BYTES(tls_static_params)];
};
struct mlx5_wqe_tls_progress_params_seg {
u8 ctx[MLX5_ST_SZ_BYTES(tls_progress_params)];
} __aligned(64);
enum {
MLX5_SET_PORT_RESET_QKEY = 0,
MLX5_SET_PORT_GUID0 = 16,
@ -750,6 +758,11 @@ static inline bool cqe_is_tunneled(struct mlx5_cqe64 *cqe)
return cqe->tls_outer_l3_tunneled & 0x1;
}
static inline u8 get_cqe_tls_offload(struct mlx5_cqe64 *cqe)
{
return (cqe->tls_outer_l3_tunneled >> 3) & 0x3;
}
enum {
CQE_L4_HDR_TYPE_NONE = 0x0,
CQE_L4_HDR_TYPE_TCP_NO_ACK = 0x1,
@ -794,6 +807,13 @@ enum {
CQE_L4_OK = 1 << 2,
};
enum {
CQE_TLS_OFFLOAD_NOT_DECRYPTED = 0x0,
CQE_TLS_OFFLOAD_DECRYPTED = 0x1,
CQE_TLS_OFFLOAD_RESYNC = 0x2,
CQE_TLS_OFFLOAD_ERROR = 0x3,
};
struct mlx5_sig_err_cqe {
u8 rsvd0[16];
__be32 expected_trans_sig;

10
sys/dev/mlx5/mlx5_en/en.h
View File

@ -207,7 +207,9 @@ typedef void (mlx5e_cq_comp_t)(struct mlx5_core_cq *, struct mlx5_eqe *);
m(+1, u64, tx_defragged, "tx_defragged", "Transmit queue defragged") \
m(+1, u64, rx_wqe_err, "rx_wqe_err", "Receive WQE errors") \
m(+1, u64, tx_jumbo_packets, "tx_jumbo_packets", "TX packets greater than 1518 octets") \
m(+1, u64, rx_steer_missed_packets, "rx_steer_missed_packets", "RX packets dropped by steering rule(s)")
m(+1, u64, rx_steer_missed_packets, "rx_steer_missed_packets", "RX packets dropped by steering rule(s)") \
m(+1, u64, rx_decrypted_ok_packets, "rx_decrypted_ok_packets", "RX packets successfully decrypted by steering rule(s)") \
m(+1, u64, rx_decrypted_error_packets, "rx_decrypted_error_packets", "RX packets not decrypted by steering rule(s)")
#define MLX5E_VPORT_STATS_NUM (0 MLX5E_VPORT_STATS(MLX5E_STATS_COUNT))
@ -608,7 +610,9 @@ struct mlx5e_port_stats_debug {
m(+1, u64, lro_bytes, "lro_bytes", "Received LRO bytes") \
m(+1, u64, sw_lro_queued, "sw_lro_queued", "Packets queued for SW LRO") \
m(+1, u64, sw_lro_flushed, "sw_lro_flushed", "Packets flushed from SW LRO") \
m(+1, u64, wqe_err, "wqe_err", "Received packets")
m(+1, u64, wqe_err, "wqe_err", "Received packets") \
m(+1, u64, decrypted_ok_packets, "decrypted_ok_packets", "Received packets successfully decrypted by steering rule(s)") \
m(+1, u64, decrypted_error_packets, "decrypted_error_packets", "Received packets not decrypted by steering rule(s)")
#define MLX5E_RQ_STATS_NUM (0 MLX5E_RQ_STATS(MLX5E_STATS_COUNT))
@ -1050,6 +1054,7 @@ struct mlx5e_xmit_args {
#include <dev/mlx5/mlx5_en/en_rl.h>
#include <dev/mlx5/mlx5_en/en_hw_tls.h>
#include <dev/mlx5/mlx5_en/en_hw_tls_rx.h>
#define MLX5E_TSTMP_PREC 10
@ -1131,6 +1136,7 @@ struct mlx5e_priv {
struct mlx5e_rl_priv_data rl;
struct mlx5e_tls tls;
struct mlx5e_tls_rx tls_rx;
struct callout tstmp_clbr;
int clbr_done;

149
sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h Normal file
View File

@ -0,0 +1,149 @@
/*-
* Copyright (c) 2021 NVIDIA corporation & affiliates.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef _MLX5_TLS_RX_H_
#define _MLX5_TLS_RX_H_
#include <linux/completion.h>
#define MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE 128
#define MLX5E_TLS_RX_RESYNC_MAX 32 /* units */
#define MLX5E_TLS_RX_NUM_MAX (1U << 11) /* packets */
#define MLX5E_TLS_RX_TAG_LOCK(tag) mtx_lock(&(tag)->mtx)
#define MLX5E_TLS_RX_TAG_UNLOCK(tag) mtx_unlock(&(tag)->mtx)
#define MLX5E_TLS_RX_STAT_INC(tag, field, num) \
counter_u64_add((tag)->tls_rx->stats.field, num)
#if ((MLX5E_TLS_RX_RESYNC_MAX * MLX5E_TLS_RX_NUM_MAX) << 14) > (1U << 30)
#error "Please lower the limits of the TLS record length database."
#endif
enum {
MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD = 0,
MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_OFFLOAD = 1,
MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_AUTHENTICATION = 2,
};
enum {
MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START = 0,
MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING = 1,
MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_SEARCHING = 2,
};
struct mlx5e_tls_rx;
struct mlx5e_tls_rx_tag {
struct m_snd_tag tag;
volatile s32 refs; /* number of pending mbufs */
uint32_t tirn; /* HW TIR context number */
uint32_t dek_index; /* HW TLS context number */
struct mlx5e_tls_rx *tls_rx; /* parent pointer */
struct mlx5_flow_rule *flow_rule;
struct mtx mtx;
struct completion progress_complete;
uint32_t state; /* see MLX5E_TLS_RX_ST_XXX */
#define MLX5E_TLS_RX_ST_INIT 0
#define MLX5E_TLS_RX_ST_SETUP 1
#define MLX5E_TLS_RX_ST_READY 2
#define MLX5E_TLS_RX_ST_FREED 3
/*
* The following fields are used to store the TCP starting
* point of TLS records in the past. When TLS records of same
* length are back to back the tcp_resync_num[] is incremented
* instead of creating new entries. This way up to
* "MLX5E_TLS_RX_RESYNC_MAX" * "MLX5E_TLS_RX_NUM_MAX" * 16
* KBytes, around 1GByte worth of TCP data, may be remembered
* in the good case. The amount of history should not exceed
* 2GBytes of TCP data, because then the TCP sequence numbers
* may wrap around.
*
* This information is used to tell if a given TCP sequence
* number is a valid TLS record or not.
*/
uint64_t rcd_resync_start; /* starting TLS record number */
uint32_t tcp_resync_start; /* starting TCP sequence number */
uint32_t tcp_resync_next; /* next expected TCP sequence number */
uint32_t tcp_resync_len[MLX5E_TLS_RX_RESYNC_MAX];
uint32_t tcp_resync_num[MLX5E_TLS_RX_RESYNC_MAX];
uint16_t tcp_resync_pc; /* producer counter for arrays above */
uint16_t tcp_resync_cc; /* consumer counter for arrays above */
struct work_struct work;
uint32_t flowid;
uint32_t flowtype;
uint32_t dek_index_ok:1;
uint32_t tcp_resync_active:1;
uint32_t tcp_resync_pending:1;
/* parameters needed */
uint8_t crypto_params[128] __aligned(4);
uint8_t rx_progress[MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE * 2];
} __aligned(MLX5E_CACHELINE_SIZE);
static inline void *
mlx5e_tls_rx_get_progress_buffer(struct mlx5e_tls_rx_tag *ptag)
{
/* return properly aligned RX buffer */
return (ptag->rx_progress +
((-(uintptr_t)ptag->rx_progress) &
(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE - 1)));
}
#define MLX5E_TLS_RX_STATS(m) \
m(+1, u64, rx_resync_ok, "rx_resync_ok", "Successful resync requests")\
m(+1, u64, rx_resync_err, "rx_resync_err", "Failed resync requests")\
m(+1, u64, rx_error, "rx_error", "Other errors")
#define MLX5E_TLS_RX_STATS_NUM (0 MLX5E_TLS_RX_STATS(MLX5E_STATS_COUNT))
struct mlx5e_tls_rx_stats {
struct sysctl_ctx_list ctx;
counter_u64_t arg[0];
MLX5E_TLS_RX_STATS(MLX5E_STATS_COUNTER)
};
struct mlx5e_tls_rx {
struct sysctl_ctx_list ctx;
struct mlx5e_tls_rx_stats stats;
struct workqueue_struct *wq;
uma_zone_t zone;
uint32_t max_resources; /* max number of resources */
volatile uint32_t num_resources; /* current number of resources */
int init; /* set when ready */
char zname[32];
};
int mlx5e_tls_rx_init(struct mlx5e_priv *);
void mlx5e_tls_rx_cleanup(struct mlx5e_priv *);
if_snd_tag_alloc_t mlx5e_tls_rx_snd_tag_alloc;
#endif /* _MLX5_TLS_RX_H_ */

1008
sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c Normal file
View File

File diff suppressed because it is too large Load Diff

26
sys/dev/mlx5/mlx5_en/mlx5_en_main.c
View File

@ -827,6 +827,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
u64 rx_wqe_err = 0;
u64 rx_packets = 0;
u64 rx_bytes = 0;
u64 rx_decrypted_error = 0;
u64 rx_decrypted_ok = 0;
u32 rx_out_of_buffer = 0;
int error;
int i;
@ -853,6 +855,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
rx_wqe_err += rq_stats->wqe_err;
rx_packets += rq_stats->packets;
rx_bytes += rq_stats->bytes;
rx_decrypted_error += rq_stats->decrypted_error_packets;
rx_decrypted_ok += rq_stats->decrypted_ok_packets;
for (j = 0; j < priv->num_tc; j++) {
sq_stats = &pch->sq[j].stats;
@ -903,6 +907,8 @@ mlx5e_update_stats_locked(struct mlx5e_priv *priv)
s->rx_wqe_err = rx_wqe_err;
s->rx_packets = rx_packets;
s->rx_bytes = rx_bytes;
s->rx_decrypted_error_packets = rx_decrypted_error;
s->rx_decrypted_ok_packets = rx_decrypted_ok;
mlx5e_grp_vnic_env_update_stats(priv);
@ -4388,6 +4394,8 @@ mlx5e_snd_tag_alloc(struct ifnet *ifp,
#ifdef KERN_TLS
case IF_SND_TAG_TYPE_TLS:
return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
case IF_SND_TAG_TYPE_TLS_RX:
return (mlx5e_tls_rx_snd_tag_alloc(ifp, params, ppmt));
#endif
default:
return (EOPNOTSUPP);
@ -4644,6 +4652,12 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
goto err_open_tirs;
}
err = mlx5e_tls_rx_init(priv);
if (err) {
if_printf(ifp, "%s: mlx5e_tls_rx_init() failed, %d\n", __func__, err);
goto err_open_flow_tables;
}
/* set default MTU */
mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
@ -4779,6 +4793,9 @@ mlx5e_create_ifp(struct mlx5_core_dev *mdev)
return (priv);
err_open_flow_tables:
mlx5e_close_flow_tables(priv);
err_open_tirs:
mlx5e_close_tirs(priv);
@ -4853,6 +4870,14 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
"Waiting for all TLS connections to terminate\n");
pause("W", hz);
}
/* wait for all TLS RX tags to get freed */
while (priv->tls_rx.init != 0 &&
uma_zone_get_cur(priv->tls_rx.zone) != 0) {
mlx5_en_err(priv->ifp,
"Waiting for all TLS RX connections to terminate\n");
pause("W", hz);
}
#endif
/* wait for all unlimited send tags to complete */
mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
@ -4887,6 +4912,7 @@ mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
ifmedia_removeall(&priv->media);
ether_ifdetach(ifp);
mlx5e_tls_rx_cleanup(priv);
mlx5e_close_flow_tables(priv);
mlx5e_close_tirs(priv);
mlx5e_close_rqts(priv);

13
sys/dev/mlx5/mlx5_en/mlx5_en_rx.c
View File

@ -372,6 +372,19 @@ mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe,
mb->m_pkthdr.rcv_tstmp = tstmp;
mb->m_flags |= M_TSTMP;
}
switch (get_cqe_tls_offload(cqe)) {
case CQE_TLS_OFFLOAD_DECRYPTED:
/* set proper checksum flag for decrypted packets */
mb->m_pkthdr.csum_flags |= CSUM_TLS_DECRYPTED;
rq->stats.decrypted_ok_packets++;
break;
case CQE_TLS_OFFLOAD_ERROR:
rq->stats.decrypted_error_packets++;
break;
default:
break;
}
}
static inline void

2
sys/dev/mlx5/mlx5_ifc.h
View File

@ -1366,7 +1366,7 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 reserved_at_480[0x1];
u8 tls_tx[0x1];
u8 reserved_at_482[0x1];
u8 tls_rx[0x1];
u8 log_max_l2_table[0x5];
u8 reserved_64[0x8];
u8 log_uar_page_sz[0x10];

1
sys/modules/mlx5en/Makefile
View File

@ -9,6 +9,7 @@ mlx5_en_main.c \
mlx5_en_tx.c \
mlx5_en_flow_table.c \
mlx5_en_hw_tls.c \
mlx5_en_hw_tls_rx.c \
mlx5_en_iq.c \
mlx5_en_rx.c \
mlx5_en_rl.c \