d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

net: add CRC implementation runtime selection

Browse Source 
This patch adds support for run-time selection of the optimal
architecture-specific CRC path, based on the supported instruction set(s)
of the CPU.

The compiler option checks have been moved from the C files to the meson
script. The rte_cpu_get_flag_enabled function is called automatically by
the library at process initialization time to determine which
instructions the CPU supports, with the most optimal supported CRC path
ultimately selected.

Signed-off-by: Mairtin o Loingsigh <mairtin.oloingsigh@intel.com>
Signed-off-by: David Coyle <david.coyle@intel.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
Reviewed-by: Jasvinder Singh <jasvinder.singh@intel.com>
Reviewed-by: Pablo de Lara <pablo.de.lara.guarch@intel.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
Acked-by: Bruce Richardson <bruce.richardson@intel.com>
This commit is contained in:
Mairtin o Loingsigh 2020-10-09 14:50:44 +01:00 committed by David Marchand
parent d9739192a7
commit ef94569cf9
7 changed files with 171 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
MAINTAINERS
View File

@ -262,7 +262,7 @@ ARM v8
M: Jerin Jacob <jerinj@marvell.com>
M: Ruifeng Wang <ruifeng.wang@arm.com>
F: lib/librte_eal/arm/include/*_64.h
F: lib/librte_net/net_crc_neon.h
F: lib/librte_net/net_crc_neon.c
F: lib/librte_acl/acl_run_neon.*
F: lib/librte_bpf/bpf_jit_arm64.c
F: lib/librte_lpm/rte_lpm_neon.h
@ -1243,8 +1243,9 @@ F: lib/librte_net/
Packet CRC
M: Jasvinder Singh <jasvinder.singh@intel.com>
F: lib/librte_net/net_crc.h
F: lib/librte_net/rte_net_crc*
F: lib/librte_net/net_crc_sse.h
F: lib/librte_net/net_crc_sse.c
F: app/test/test_crc.c
IP fragmentation & reassembly

4
doc/guides/rel_notes/release_20_11.rst
View File

@ -62,6 +62,10 @@ New Features
The functions are provided as a generic stubs and
x86 specific implementation.
* **Updated CRC modules of the net library.**
* Added runtime selection of the optimal architecture-specific CRC path.
* **Added the FEC API, for a generic FEC query and config.**
Added the FEC API which provides functions for query FEC capabilities and

34
lib/librte_net/meson.build
View File

@ -1,5 +1,5 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2017 Intel Corporation
# Copyright(c) 2017-2020 Intel Corporation
headers = files('rte_ip.h',
'rte_tcp.h',
@ -20,3 +20,35 @@ headers = files('rte_ip.h',
sources = files('rte_arp.c', 'rte_ether.c', 'rte_net.c', 'rte_net_crc.c')
deps += ['mbuf']
if dpdk_conf.has('RTE_ARCH_X86_64')
net_crc_sse42_cpu_support = (
cc.get_define('__PCLMUL__', args: machine_args) != '')
net_crc_sse42_cc_support = (
cc.has_argument('-mpclmul') and cc.has_argument('-maes'))
build_static_net_crc_sse42_lib = 0
if net_crc_sse42_cpu_support == true
sources += files('net_crc_sse.c')
cflags += ['-DCC_X86_64_SSE42_PCLMULQDQ_SUPPORT']
elif net_crc_sse42_cc_support == true
build_static_net_crc_sse42_lib = 1
net_crc_sse42_lib_cflags = ['-mpclmul', '-maes']
cflags += ['-DCC_X86_64_SSE42_PCLMULQDQ_SUPPORT']
endif
if build_static_net_crc_sse42_lib == 1
net_crc_sse42_lib = static_library(
'net_crc_sse42_lib',
'net_crc_sse.c',
dependencies: static_rte_eal,
c_args: [cflags,
net_crc_sse42_lib_cflags])
objs += net_crc_sse42_lib.extract_objects('net_crc_sse.c')
endif
elif (dpdk_conf.has('RTE_ARCH_ARM64') and
cc.get_define('__ARM_FEATURE_CRYPTO', args: machine_args) != '')
sources += files('net_crc_neon.c')
cflags += ['-DCC_ARM64_NEON_PMULL_SUPPORT']
endif

34
lib/librte_net/net_crc.h Normal file
View File

@ -0,0 +1,34 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2020 Intel Corporation
*/
#ifndef _NET_CRC_H_
#define _NET_CRC_H_
/*
* Different implementations of CRC
*/
/* SSE4.2 */
void
rte_net_crc_sse42_init(void);
uint32_t
rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len);
uint32_t
rte_crc32_eth_sse42_handler(const uint8_t *data, uint32_t data_len);
/* NEON */
void
rte_net_crc_neon_init(void);
uint32_t
rte_crc16_ccitt_neon_handler(const uint8_t *data, uint32_t data_len);
uint32_t
rte_crc32_eth_neon_handler(const uint8_t *data, uint32_t data_len);
#endif /* _NET_CRC_H_ */

26
lib/librte_net/net_crc_neon.h → lib/librte_net/net_crc_neon.c
View File

@ -2,17 +2,15 @@
* Copyright(c) 2017 Cavium, Inc
*/
#ifndef _NET_CRC_NEON_H_
#define _NET_CRC_NEON_H_
#include <string.h>
#include <rte_common.h>
#include <rte_branch_prediction.h>
#include <rte_net_crc.h>
#include <rte_vect.h>
#include <rte_cpuflags.h>
#ifdef __cplusplus
extern "C" {
#endif
#include "net_crc.h"
/** PMULL CRC computation context structure */
struct crc_pmull_ctx {
@ -218,7 +216,7 @@ crc32_eth_calc_pmull(
return n;
}
static inline void
void
rte_net_crc_neon_init(void)
{
/* Initialize CRC16 data */
@ -242,9 +240,8 @@ rte_net_crc_neon_init(void)
crc32_eth_pmull.rk7_rk8 = vld1q_u64(eth_k7_k8);
}
static inline uint32_t
rte_crc16_ccitt_neon_handler(const uint8_t *data,
uint32_t data_len)
uint32_t
rte_crc16_ccitt_neon_handler(const uint8_t *data, uint32_t data_len)
{
return (uint16_t)~crc32_eth_calc_pmull(data,
data_len,
@ -252,18 +249,11 @@ rte_crc16_ccitt_neon_handler(const uint8_t *data,
&crc16_ccitt_pmull);
}
static inline uint32_t
rte_crc32_eth_neon_handler(const uint8_t *data,
uint32_t data_len)
uint32_t
rte_crc32_eth_neon_handler(const uint8_t *data, uint32_t data_len)
{
return ~crc32_eth_calc_pmull(data,
data_len,
0xffffffffUL,
&crc32_eth_pmull);
}
#ifdef __cplusplus
}
#endif
#endif /* _NET_CRC_NEON_H_ */

34
lib/librte_net/net_crc_sse.h → lib/librte_net/net_crc_sse.c
View File

@ -1,18 +1,16 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Intel Corporation
* Copyright(c) 2017-2020 Intel Corporation
*/
#ifndef _RTE_NET_CRC_SSE_H_
#define _RTE_NET_CRC_SSE_H_
#include <string.h>
#include <rte_common.h>
#include <rte_branch_prediction.h>
#include <rte_cpuflags.h>
#include "net_crc.h"
#include <x86intrin.h>
#include <cpuid.h>
#ifdef __cplusplus
extern "C" {
#endif
/** PCLMULQDQ CRC computation context structure */
struct crc_pclmulqdq_ctx {
@ -259,8 +257,7 @@ crc32_eth_calc_pclmulqdq(
return n;
}
static inline void
void
rte_net_crc_sse42_init(void)
{
uint64_t k1, k2, k5, k6;
@ -303,12 +300,10 @@ rte_net_crc_sse42_init(void)
* use other data types such as float, double, etc.
*/
_mm_empty();
}
static inline uint32_t
rte_crc16_ccitt_sse42_handler(const uint8_t *data,
uint32_t data_len)
uint32_t
rte_crc16_ccitt_sse42_handler(const uint8_t *data, uint32_t data_len)
{
/** return 16-bit CRC value */
return (uint16_t)~crc32_eth_calc_pclmulqdq(data,
@ -317,18 +312,11 @@ rte_crc16_ccitt_sse42_handler(const uint8_t *data,
&crc16_ccitt_pclmulqdq);
}
static inline uint32_t
rte_crc32_eth_sse42_handler(const uint8_t *data,
uint32_t data_len)
uint32_t
rte_crc32_eth_sse42_handler(const uint8_t *data, uint32_t data_len)
{
return ~crc32_eth_calc_pclmulqdq(data,
data_len,
0xffffffffUL,
&crc32_eth_pclmulqdq);
}
#ifdef __cplusplus
}
#endif
#endif /* _RTE_NET_CRC_SSE_H_ */

116
lib/librte_net/rte_net_crc.c
View File

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2017 Intel Corporation
* Copyright(c) 2017-2020 Intel Corporation
*/
#include <stddef.h>
@ -10,17 +10,7 @@
#include <rte_common.h>
#include <rte_net_crc.h>
#if defined(RTE_ARCH_X86_64) && defined(__PCLMUL__)
#define X86_64_SSE42_PCLMULQDQ 1
#elif defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRYPTO)
#define ARM64_NEON_PMULL 1
#endif
#ifdef X86_64_SSE42_PCLMULQDQ
#include <net_crc_sse.h>
#elif defined ARM64_NEON_PMULL
#include <net_crc_neon.h>
#endif
#include "net_crc.h"
/** CRC polynomials */
#define CRC32_ETH_POLYNOMIAL 0x04c11db7UL
@ -41,25 +31,27 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len);
typedef uint32_t
(*rte_net_crc_handler)(const uint8_t *data, uint32_t data_len);
static rte_net_crc_handler *handlers;
static const rte_net_crc_handler *handlers;
static rte_net_crc_handler handlers_scalar[] = {
static const rte_net_crc_handler handlers_scalar[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_handler,
};
#ifdef X86_64_SSE42_PCLMULQDQ
static rte_net_crc_handler handlers_sse42[] = {
#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
static const rte_net_crc_handler handlers_sse42[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_sse42_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_sse42_handler,
};
#elif defined ARM64_NEON_PMULL
static rte_net_crc_handler handlers_neon[] = {
#endif
#ifdef CC_ARM64_NEON_PMULL_SUPPORT
static const rte_net_crc_handler handlers_neon[] = {
[RTE_NET_CRC16_CCITT] = rte_crc16_ccitt_neon_handler,
[RTE_NET_CRC32_ETH] = rte_crc32_eth_neon_handler,
};
#endif
/* Scalar handling */
/**
* Reflect the bits about the middle
*
@ -142,29 +134,82 @@ rte_crc32_eth_handler(const uint8_t *data, uint32_t data_len)
crc32_eth_lut);
}
/* SSE4.2/PCLMULQDQ handling */
#define SSE42_PCLMULQDQ_CPU_SUPPORTED \
rte_cpu_get_flag_enabled(RTE_CPUFLAG_PCLMULQDQ)
static const rte_net_crc_handler *
sse42_pclmulqdq_get_handlers(void)
{
#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
if (SSE42_PCLMULQDQ_CPU_SUPPORTED)
return handlers_sse42;
#endif
return NULL;
}
static uint8_t
sse42_pclmulqdq_init(void)
{
#ifdef CC_X86_64_SSE42_PCLMULQDQ_SUPPORT
if (SSE42_PCLMULQDQ_CPU_SUPPORTED) {
rte_net_crc_sse42_init();
return 1;
}
#endif
return 0;
}
/* NEON/PMULL handling */
#define NEON_PMULL_CPU_SUPPORTED \
rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)
static const rte_net_crc_handler *
neon_pmull_get_handlers(void)
{
#ifdef CC_ARM64_NEON_PMULL_SUPPORT
if (NEON_PMULL_CPU_SUPPORTED)
return handlers_neon;
#endif
return NULL;
}
static uint8_t
neon_pmull_init(void)
{
#ifdef CC_ARM64_NEON_PMULL_SUPPORT
if (NEON_PMULL_CPU_SUPPORTED) {
rte_net_crc_neon_init();
return 1;
}
#endif
return 0;
}
/* Public API */
void
rte_net_crc_set_alg(enum rte_net_crc_alg alg)
{
handlers = NULL;
switch (alg) {
#ifdef X86_64_SSE42_PCLMULQDQ
case RTE_NET_CRC_SSE42:
handlers = handlers_sse42;
break;
#elif defined ARM64_NEON_PMULL
/* fall-through */
handlers = sse42_pclmulqdq_get_handlers();
break; /* for x86, always break here */
case RTE_NET_CRC_NEON:
if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
handlers = handlers_neon;
break;
}
#endif
handlers = neon_pmull_get_handlers();
/* fall-through */
case RTE_NET_CRC_SCALAR:
/* fall-through */
default:
handlers = handlers_scalar;
break;
}
if (handlers == NULL)
handlers = handlers_scalar;
}
uint32_t
@ -188,15 +233,10 @@ RTE_INIT(rte_net_crc_init)
rte_net_crc_scalar_init();
#ifdef X86_64_SSE42_PCLMULQDQ
alg = RTE_NET_CRC_SSE42;
rte_net_crc_sse42_init();
#elif defined ARM64_NEON_PMULL
if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_PMULL)) {
if (sse42_pclmulqdq_init())
alg = RTE_NET_CRC_SSE42;
if (neon_pmull_init())
alg = RTE_NET_CRC_NEON;
rte_net_crc_neon_init();
}
#endif
rte_net_crc_set_alg(alg);
}