acl: fix build and runtime for default target
Make ACL library to build/work on 'default' architecture: - make rte_acl_classify_scalar really scalar (make sure it wouldn't use sse4 instrincts through resolve_priority()). - Provide two versions of rte_acl_classify code path: rte_acl_classify_sse() - could be build and used only on systems with sse4.2 and upper, return -ENOTSUP on lower arch. rte_acl_classify_scalar() - a slower version, but could be build and used on all systems. - Addition of a new function rte_acl_classify_alg. This function lets you specify an enum value to override the acl contexts default algorithm when doing a classification. This allows an application to specify a classification algorithm without needing to publicize each method. I know there was concern over keeping those methods public, but we don't have a static ABI at the moment, so this seems to me a reasonable thing to do, as it gives us less of an ABI surface to worry about. - keep common code shared between these two codepaths. Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com> Acked-by: Neil Horman <nhorman@tuxdriver.com>
This commit is contained in:
Konstantin Ananyev
Thomas Monjalon
parent
8fd8bebc05
commit
074f54ad03
@ -772,6 +772,15 @@ acx_init(void)
|
||||
if (config.acx == NULL)
|
||||
rte_exit(rte_errno, "failed to create ACL context\n");
|
||||
|
||||
/* set default classify method to scalar for this context. */
|
||||
if (config.scalar) {
|
||||
ret = rte_acl_set_ctx_classify(config.acx,
|
||||
RTE_ACL_CLASSIFY_SCALAR);
|
||||
if (ret != 0)
|
||||
rte_exit(ret, "failed to setup classify method "
|
||||
"for ACL context\n");
|
||||
}
|
||||
|
||||
/* add ACL rules. */
|
||||
f = fopen(config.rule_file, "r");
|
||||
if (f == NULL)
|
||||
@ -780,7 +789,7 @@ acx_init(void)
|
||||
|
||||
ret = add_cb_rules(f, config.acx);
|
||||
if (ret != 0)
|
||||
rte_exit(rte_errno, "failed to add rules into ACL context\n");
|
||||
rte_exit(ret, "failed to add rules into ACL context\n");
|
||||
|
||||
fclose(f);
|
||||
|
||||
@ -815,13 +824,8 @@ search_ip5tuples_once(uint32_t categories, uint32_t step, int scalar)
|
||||
v += config.trace_sz;
|
||||
}
|
||||
|
||||
if (scalar != 0)
|
||||
ret = rte_acl_classify_scalar(config.acx, data,
|
||||
results, n, categories);
|
||||
|
||||
else
|
||||
ret = rte_acl_classify(config.acx, data,
|
||||
results, n, categories);
|
||||
ret = rte_acl_classify(config.acx, data, results,
|
||||
n, categories);
|
||||
|
||||
if (ret != 0)
|
||||
rte_exit(ret, "classify for ipv%c_5tuples returns %d\n",
|
||||
|
||||
@ -146,8 +146,9 @@ test_classify_run(struct rte_acl_ctx *acx)
|
||||
}
|
||||
|
||||
/* make a quick check for scalar */
|
||||
ret = rte_acl_classify_scalar(acx, data, results,
|
||||
RTE_DIM(acl_test_data), RTE_ACL_MAX_CATEGORIES);
|
||||
ret = rte_acl_classify_alg(acx, data, results,
|
||||
RTE_DIM(acl_test_data), RTE_ACL_MAX_CATEGORIES,
|
||||
RTE_ACL_CLASSIFY_SCALAR);
|
||||
if (ret != 0) {
|
||||
printf("Line %i: SSE classify failed!\n", __LINE__);
|
||||
goto err;
|
||||
@ -341,8 +342,8 @@ test_invalid_layout(void)
|
||||
}
|
||||
|
||||
/* classify tuples */
|
||||
ret = rte_acl_classify(acx, data, results,
|
||||
RTE_DIM(results), 1);
|
||||
ret = rte_acl_classify_alg(acx, data, results,
|
||||
RTE_DIM(results), 1, RTE_ACL_CLASSIFY_SCALAR);
|
||||
if (ret != 0) {
|
||||
printf("Line %i: SSE classify failed!\n", __LINE__);
|
||||
rte_acl_free(acx);
|
||||
@ -360,8 +361,9 @@ test_invalid_layout(void)
|
||||
}
|
||||
|
||||
/* classify tuples (scalar) */
|
||||
ret = rte_acl_classify_scalar(acx, data, results,
|
||||
RTE_DIM(results), 1);
|
||||
ret = rte_acl_classify_alg(acx, data, results, RTE_DIM(results), 1,
|
||||
RTE_ACL_CLASSIFY_SCALAR);
|
||||
|
||||
if (ret != 0) {
|
||||
printf("Line %i: Scalar classify failed!\n", __LINE__);
|
||||
rte_acl_free(acx);
|
||||
@ -848,7 +850,8 @@ test_invalid_parameters(void)
|
||||
/* scalar classify test */
|
||||
|
||||
/* cover zero categories in classify (should not fail) */
|
||||
result = rte_acl_classify_scalar(acx, NULL, NULL, 0, 0);
|
||||
result = rte_acl_classify_alg(acx, NULL, NULL, 0, 0,
|
||||
RTE_ACL_CLASSIFY_SCALAR);
|
||||
if (result != 0) {
|
||||
printf("Line %i: Scalar classify with zero categories "
|
||||
"failed!\n", __LINE__);
|
||||
@ -857,7 +860,7 @@ test_invalid_parameters(void)
|
||||
}
|
||||
|
||||
/* cover invalid but positive categories in classify */
|
||||
result = rte_acl_classify_scalar(acx, NULL, NULL, 0, 3);
|
||||
result = rte_acl_classify(acx, NULL, NULL, 0, 3);
|
||||
if (result == 0) {
|
||||
printf("Line %i: Scalar classify with 3 categories "
|
||||
"should have failed!\n", __LINE__);
|
||||
|
||||
@ -278,15 +278,6 @@ send_single_packet(struct rte_mbuf *m, uint8_t port);
|
||||
(in) = end + 1; \
|
||||
} while (0)
|
||||
|
||||
#define CLASSIFY(context, data, res, num, cat) do { \
|
||||
if (scalar) \
|
||||
rte_acl_classify_scalar((context), (data), \
|
||||
(res), (num), (cat)); \
|
||||
else \
|
||||
rte_acl_classify((context), (data), \
|
||||
(res), (num), (cat)); \
|
||||
} while (0)
|
||||
|
||||
/*
|
||||
* ACL rules should have higher priorities than route ones to ensure ACL rule
|
||||
* always be found when input packets have multi-matches in the database.
|
||||
@ -1216,6 +1207,11 @@ setup_acl(struct rte_acl_rule *route_base,
|
||||
if ((context = rte_acl_create(&acl_param)) == NULL)
|
||||
rte_exit(EXIT_FAILURE, "Failed to create ACL context\n");
|
||||
|
||||
if (parm_config.scalar && rte_acl_set_ctx_classify(context,
|
||||
RTE_ACL_CLASSIFY_SCALAR) != 0)
|
||||
rte_exit(EXIT_FAILURE,
|
||||
"Failed to setup classify method for ACL context\n");
|
||||
|
||||
if (rte_acl_add_rules(context, route_base, route_num) < 0)
|
||||
rte_exit(EXIT_FAILURE, "add rules failed\n");
|
||||
|
||||
@ -1436,10 +1432,8 @@ main_loop(__attribute__((unused)) void *dummy)
|
||||
int socketid;
|
||||
const uint64_t drain_tsc = (rte_get_tsc_hz() + US_PER_S - 1)
|
||||
/ US_PER_S * BURST_TX_DRAIN_US;
|
||||
int scalar = parm_config.scalar;
|
||||
|
||||
prev_tsc = 0;
|
||||
|
||||
lcore_id = rte_lcore_id();
|
||||
qconf = &lcore_conf[lcore_id];
|
||||
socketid = rte_lcore_to_socket_id(lcore_id);
|
||||
@ -1503,7 +1497,8 @@ main_loop(__attribute__((unused)) void *dummy)
|
||||
nb_rx);
|
||||
|
||||
if (acl_search.num_ipv4) {
|
||||
CLASSIFY(acl_config.acx_ipv4[socketid],
|
||||
rte_acl_classify(
|
||||
acl_config.acx_ipv4[socketid],
|
||||
acl_search.data_ipv4,
|
||||
acl_search.res_ipv4,
|
||||
acl_search.num_ipv4,
|
||||
@ -1515,7 +1510,8 @@ main_loop(__attribute__((unused)) void *dummy)
|
||||
}
|
||||
|
||||
if (acl_search.num_ipv6) {
|
||||
CLASSIFY(acl_config.acx_ipv6[socketid],
|
||||
rte_acl_classify(
|
||||
acl_config.acx_ipv6[socketid],
|
||||
acl_search.data_ipv6,
|
||||
acl_search.res_ipv6,
|
||||
acl_search.num_ipv6,
|
||||
|
||||
@ -43,7 +43,10 @@ SRCS-$(CONFIG_RTE_LIBRTE_ACL) += tb_mem.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += rte_acl.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_bld.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_gen.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_scalar.c
|
||||
SRCS-$(CONFIG_RTE_LIBRTE_ACL) += acl_run_sse.c
|
||||
|
||||
CFLAGS_acl_run_sse.o += -msse4.1
|
||||
|
||||
# install this header file
|
||||
SYMLINK-$(CONFIG_RTE_LIBRTE_ACL)-include := rte_acl_osdep.h
|
||||
|
||||
@ -153,6 +153,7 @@ struct rte_acl_ctx {
|
||||
/** Name of the ACL context. */
|
||||
int32_t socket_id;
|
||||
/** Socket ID to allocate memory from. */
|
||||
enum rte_acl_classify_alg alg;
|
||||
void *rules;
|
||||
uint32_t max_rules;
|
||||
uint32_t rule_sz;
|
||||
@ -174,6 +175,20 @@ int rte_acl_gen(struct rte_acl_ctx *ctx, struct rte_acl_trie *trie,
|
||||
struct rte_acl_bld_trie *node_bld_trie, uint32_t num_tries,
|
||||
uint32_t num_categories, uint32_t data_index_sz, int match_num);
|
||||
|
||||
typedef int (*rte_acl_classify_t)
|
||||
(const struct rte_acl_ctx *, const uint8_t **, uint32_t *, uint32_t, uint32_t);
|
||||
|
||||
/*
|
||||
* Different implementations of ACL classify.
|
||||
*/
|
||||
int
|
||||
rte_acl_classify_scalar(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories);
|
||||
|
||||
int
|
||||
rte_acl_classify_sse(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif /* __cplusplus */
|
||||
|
||||
@ -31,7 +31,6 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <nmmintrin.h>
|
||||
#include <rte_acl.h>
|
||||
#include "tb_mem.h"
|
||||
#include "acl.h"
|
||||
@ -1480,8 +1479,8 @@ acl_calc_wildness(struct rte_acl_build_rule *head,
|
||||
|
||||
switch (rule->config->defs[n].type) {
|
||||
case RTE_ACL_FIELD_TYPE_BITMASK:
|
||||
wild = (size -
|
||||
_mm_popcnt_u32(fld->mask_range.u8)) /
|
||||
wild = (size - __builtin_popcount(
|
||||
fld->mask_range.u8)) /
|
||||
size;
|
||||
break;
|
||||
|
||||
|
||||
268
lib/librte_acl/acl_run.h
Normal file
268
lib/librte_acl/acl_run.h
Normal file
@ -0,0 +1,268 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#ifndef _ACL_RUN_H_
|
||||
#define _ACL_RUN_H_
|
||||
|
||||
#include <rte_acl.h>
|
||||
#include "acl_vect.h"
|
||||
#include "acl.h"
|
||||
|
||||
#define MAX_SEARCHES_SSE8 8
|
||||
#define MAX_SEARCHES_SSE4 4
|
||||
#define MAX_SEARCHES_SSE2 2
|
||||
#define MAX_SEARCHES_SCALAR 2
|
||||
|
||||
#define GET_NEXT_4BYTES(prm, idx) \
|
||||
(*((const int32_t *)((prm)[(idx)].data + *(prm)[idx].data_index++)))
|
||||
|
||||
|
||||
#define RTE_ACL_NODE_INDEX ((uint32_t)~RTE_ACL_NODE_TYPE)
|
||||
|
||||
#define SCALAR_QRANGE_MULT 0x01010101
|
||||
#define SCALAR_QRANGE_MASK 0x7f7f7f7f
|
||||
#define SCALAR_QRANGE_MIN 0x80808080
|
||||
|
||||
/*
|
||||
* Structure to manage N parallel trie traversals.
|
||||
* The runtime trie traversal routines can process 8, 4, or 2 tries
|
||||
* in parallel. Each packet may require multiple trie traversals (up to 4).
|
||||
* This structure is used to fill the slots (0 to n-1) for parallel processing
|
||||
* with the trie traversals needed for each packet.
|
||||
*/
|
||||
struct acl_flow_data {
|
||||
uint32_t num_packets;
|
||||
/* number of packets processed */
|
||||
uint32_t started;
|
||||
/* number of trie traversals in progress */
|
||||
uint32_t trie;
|
||||
/* current trie index (0 to N-1) */
|
||||
uint32_t cmplt_size;
|
||||
uint32_t total_packets;
|
||||
uint32_t categories;
|
||||
/* number of result categories per packet. */
|
||||
/* maximum number of packets to process */
|
||||
const uint64_t *trans;
|
||||
const uint8_t **data;
|
||||
uint32_t *results;
|
||||
struct completion *last_cmplt;
|
||||
struct completion *cmplt_array;
|
||||
};
|
||||
|
||||
/*
|
||||
* Structure to maintain running results for
|
||||
* a single packet (up to 4 tries).
|
||||
*/
|
||||
struct completion {
|
||||
uint32_t *results; /* running results. */
|
||||
int32_t priority[RTE_ACL_MAX_CATEGORIES]; /* running priorities. */
|
||||
uint32_t count; /* num of remaining tries */
|
||||
/* true for allocated struct */
|
||||
} __attribute__((aligned(XMM_SIZE)));
|
||||
|
||||
/*
|
||||
* One parms structure for each slot in the search engine.
|
||||
*/
|
||||
struct parms {
|
||||
const uint8_t *data;
|
||||
/* input data for this packet */
|
||||
const uint32_t *data_index;
|
||||
/* data indirection for this trie */
|
||||
struct completion *cmplt;
|
||||
/* completion data for this packet */
|
||||
};
|
||||
|
||||
/*
|
||||
* Define an global idle node for unused engine slots
|
||||
*/
|
||||
static const uint32_t idle[UINT8_MAX + 1];
|
||||
|
||||
/*
|
||||
* Allocate a completion structure to manage the tries for a packet.
|
||||
*/
|
||||
static inline struct completion *
|
||||
alloc_completion(struct completion *p, uint32_t size, uint32_t tries,
|
||||
uint32_t *results)
|
||||
{
|
||||
uint32_t n;
|
||||
|
||||
for (n = 0; n < size; n++) {
|
||||
|
||||
if (p[n].count == 0) {
|
||||
|
||||
/* mark as allocated and set number of tries. */
|
||||
p[n].count = tries;
|
||||
p[n].results = results;
|
||||
return &(p[n]);
|
||||
}
|
||||
}
|
||||
|
||||
/* should never get here */
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Resolve priority for a single result trie.
|
||||
*/
|
||||
static inline void
|
||||
resolve_single_priority(uint64_t transition, int n,
|
||||
const struct rte_acl_ctx *ctx, struct parms *parms,
|
||||
const struct rte_acl_match_results *p)
|
||||
{
|
||||
if (parms[n].cmplt->count == ctx->num_tries ||
|
||||
parms[n].cmplt->priority[0] <=
|
||||
p[transition].priority[0]) {
|
||||
|
||||
parms[n].cmplt->priority[0] = p[transition].priority[0];
|
||||
parms[n].cmplt->results[0] = p[transition].results[0];
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Routine to fill a slot in the parallel trie traversal array (parms) from
|
||||
* the list of packets (flows).
|
||||
*/
|
||||
static inline uint64_t
|
||||
acl_start_next_trie(struct acl_flow_data *flows, struct parms *parms, int n,
|
||||
const struct rte_acl_ctx *ctx)
|
||||
{
|
||||
uint64_t transition;
|
||||
|
||||
/* if there are any more packets to process */
|
||||
if (flows->num_packets < flows->total_packets) {
|
||||
parms[n].data = flows->data[flows->num_packets];
|
||||
parms[n].data_index = ctx->trie[flows->trie].data_index;
|
||||
|
||||
/* if this is the first trie for this packet */
|
||||
if (flows->trie == 0) {
|
||||
flows->last_cmplt = alloc_completion(flows->cmplt_array,
|
||||
flows->cmplt_size, ctx->num_tries,
|
||||
flows->results +
|
||||
flows->num_packets * flows->categories);
|
||||
}
|
||||
|
||||
/* set completion parameters and starting index for this slot */
|
||||
parms[n].cmplt = flows->last_cmplt;
|
||||
transition =
|
||||
flows->trans[parms[n].data[*parms[n].data_index++] +
|
||||
ctx->trie[flows->trie].root_index];
|
||||
|
||||
/*
|
||||
* if this is the last trie for this packet,
|
||||
* then setup next packet.
|
||||
*/
|
||||
flows->trie++;
|
||||
if (flows->trie >= ctx->num_tries) {
|
||||
flows->trie = 0;
|
||||
flows->num_packets++;
|
||||
}
|
||||
|
||||
/* keep track of number of active trie traversals */
|
||||
flows->started++;
|
||||
|
||||
/* no more tries to process, set slot to an idle position */
|
||||
} else {
|
||||
transition = ctx->idle;
|
||||
parms[n].data = (const uint8_t *)idle;
|
||||
parms[n].data_index = idle;
|
||||
}
|
||||
return transition;
|
||||
}
|
||||
|
||||
static inline void
|
||||
acl_set_flow(struct acl_flow_data *flows, struct completion *cmplt,
|
||||
uint32_t cmplt_size, const uint8_t **data, uint32_t *results,
|
||||
uint32_t data_num, uint32_t categories, const uint64_t *trans)
|
||||
{
|
||||
flows->num_packets = 0;
|
||||
flows->started = 0;
|
||||
flows->trie = 0;
|
||||
flows->last_cmplt = NULL;
|
||||
flows->cmplt_array = cmplt;
|
||||
flows->total_packets = data_num;
|
||||
flows->categories = categories;
|
||||
flows->cmplt_size = cmplt_size;
|
||||
flows->data = data;
|
||||
flows->results = results;
|
||||
flows->trans = trans;
|
||||
}
|
||||
|
||||
typedef void (*resolve_priority_t)
|
||||
(uint64_t transition, int n, const struct rte_acl_ctx *ctx,
|
||||
struct parms *parms, const struct rte_acl_match_results *p,
|
||||
uint32_t categories);
|
||||
|
||||
/*
|
||||
* Detect matches. If a match node transition is found, then this trie
|
||||
* traversal is complete and fill the slot with the next trie
|
||||
* to be processed.
|
||||
*/
|
||||
static inline uint64_t
|
||||
acl_match_check(uint64_t transition, int slot,
|
||||
const struct rte_acl_ctx *ctx, struct parms *parms,
|
||||
struct acl_flow_data *flows, resolve_priority_t resolve_priority)
|
||||
{
|
||||
const struct rte_acl_match_results *p;
|
||||
|
||||
p = (const struct rte_acl_match_results *)
|
||||
(flows->trans + ctx->match_index);
|
||||
|
||||
if (transition & RTE_ACL_NODE_MATCH) {
|
||||
|
||||
/* Remove flags from index and decrement active traversals */
|
||||
transition &= RTE_ACL_NODE_INDEX;
|
||||
flows->started--;
|
||||
|
||||
/* Resolve priorities for this trie and running results */
|
||||
if (flows->categories == 1)
|
||||
resolve_single_priority(transition, slot, ctx,
|
||||
parms, p);
|
||||
else
|
||||
resolve_priority(transition, slot, ctx, parms,
|
||||
p, flows->categories);
|
||||
|
||||
/* Count down completed tries for this search request */
|
||||
parms[slot].cmplt->count--;
|
||||
|
||||
/* Fill the slot with the next trie or idle trie */
|
||||
transition = acl_start_next_trie(flows, parms, slot, ctx);
|
||||
|
||||
} else if (transition == ctx->idle) {
|
||||
/* reset indirection table for idle slots */
|
||||
parms[slot].data_index = idle;
|
||||
}
|
||||
|
||||
return transition;
|
||||
}
|
||||
|
||||
#endif /* _ACL_RUN_H_ */
|
||||
193
lib/librte_acl/acl_run_scalar.c
Normal file
193
lib/librte_acl/acl_run_scalar.c
Normal file
@ -0,0 +1,193 @@
|
||||
/*-
|
||||
* BSD LICENSE
|
||||
*
|
||||
* Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
|
||||
* All rights reserved.
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
*
|
||||
* * Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* * Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in
|
||||
* the documentation and/or other materials provided with the
|
||||
* distribution.
|
||||
* * Neither the name of Intel Corporation nor the names of its
|
||||
* contributors may be used to endorse or promote products derived
|
||||
* from this software without specific prior written permission.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||||
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||||
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||||
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||||
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||||
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||||
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||||
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||||
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||||
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include "acl_run.h"
|
||||
|
||||
/*
|
||||
* Resolve priority for multiple results (scalar version).
|
||||
* This consists comparing the priority of the current traversal with the
|
||||
* running set of results for the packet.
|
||||
* For each result, keep a running array of the result (rule number) and
|
||||
* its priority for each category.
|
||||
*/
|
||||
static inline void
|
||||
resolve_priority_scalar(uint64_t transition, int n,
|
||||
const struct rte_acl_ctx *ctx, struct parms *parms,
|
||||
const struct rte_acl_match_results *p, uint32_t categories)
|
||||
{
|
||||
uint32_t i;
|
||||
int32_t *saved_priority;
|
||||
uint32_t *saved_results;
|
||||
const int32_t *priority;
|
||||
const uint32_t *results;
|
||||
|
||||
saved_results = parms[n].cmplt->results;
|
||||
saved_priority = parms[n].cmplt->priority;
|
||||
|
||||
/* results and priorities for completed trie */
|
||||
results = p[transition].results;
|
||||
priority = p[transition].priority;
|
||||
|
||||
/* if this is not the first completed trie */
|
||||
if (parms[n].cmplt->count != ctx->num_tries) {
|
||||
for (i = 0; i < categories; i += RTE_ACL_RESULTS_MULTIPLIER) {
|
||||
|
||||
if (saved_priority[i] <= priority[i]) {
|
||||
saved_priority[i] = priority[i];
|
||||
saved_results[i] = results[i];
|
||||
}
|
||||
if (saved_priority[i + 1] <= priority[i + 1]) {
|
||||
saved_priority[i + 1] = priority[i + 1];
|
||||
saved_results[i + 1] = results[i + 1];
|
||||
}
|
||||
if (saved_priority[i + 2] <= priority[i + 2]) {
|
||||
saved_priority[i + 2] = priority[i + 2];
|
||||
saved_results[i + 2] = results[i + 2];
|
||||
}
|
||||
if (saved_priority[i + 3] <= priority[i + 3]) {
|
||||
saved_priority[i + 3] = priority[i + 3];
|
||||
saved_results[i + 3] = results[i + 3];
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (i = 0; i < categories; i += RTE_ACL_RESULTS_MULTIPLIER) {
|
||||
saved_priority[i] = priority[i];
|
||||
saved_priority[i + 1] = priority[i + 1];
|
||||
saved_priority[i + 2] = priority[i + 2];
|
||||
saved_priority[i + 3] = priority[i + 3];
|
||||
|
||||
saved_results[i] = results[i];
|
||||
saved_results[i + 1] = results[i + 1];
|
||||
saved_results[i + 2] = results[i + 2];
|
||||
saved_results[i + 3] = results[i + 3];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* When processing the transition, rather than using if/else
|
||||
* construct, the offset is calculated for DFA and QRANGE and
|
||||
* then conditionally added to the address based on node type.
|
||||
* This is done to avoid branch mis-predictions. Since the
|
||||
* offset is rather simple calculation it is more efficient
|
||||
* to do the calculation and do a condition move rather than
|
||||
* a conditional branch to determine which calculation to do.
|
||||
*/
|
||||
static inline uint32_t
|
||||
scan_forward(uint32_t input, uint32_t max)
|
||||
{
|
||||
return (input == 0) ? max : rte_bsf32(input);
|
||||
}
|
||||
|
||||
static inline uint64_t
|
||||
scalar_transition(const uint64_t *trans_table, uint64_t transition,
|
||||
uint8_t input)
|
||||
{
|
||||
uint32_t addr, index, ranges, x, a, b, c;
|
||||
|
||||
/* break transition into component parts */
|
||||
ranges = transition >> (sizeof(index) * CHAR_BIT);
|
||||
|
||||
/* calc address for a QRANGE node */
|
||||
c = input * SCALAR_QRANGE_MULT;
|
||||
a = ranges | SCALAR_QRANGE_MIN;
|
||||
index = transition & ~RTE_ACL_NODE_INDEX;
|
||||
a -= (c & SCALAR_QRANGE_MASK);
|
||||
b = c & SCALAR_QRANGE_MIN;
|
||||
addr = transition ^ index;
|
||||
a &= SCALAR_QRANGE_MIN;
|
||||
a ^= (ranges ^ b) & (a ^ b);
|
||||
x = scan_forward(a, 32) >> 3;
|
||||
addr += (index == RTE_ACL_NODE_DFA) ? input : x;
|
||||
|
||||
/* pickup next transition */
|
||||
transition = *(trans_table + addr);
|
||||
return transition;
|
||||
}
|
||||
|
||||
int
|
||||
rte_acl_classify_scalar(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories)
|
||||
{
|
||||
int n;
|
||||
uint64_t transition0, transition1;
|
||||
uint32_t input0, input1;
|
||||
struct acl_flow_data flows;
|
||||
uint64_t index_array[MAX_SEARCHES_SCALAR];
|
||||
struct completion cmplt[MAX_SEARCHES_SCALAR];
|
||||
struct parms parms[MAX_SEARCHES_SCALAR];
|
||||
|
||||
if (categories != 1 &&
|
||||
((RTE_ACL_RESULTS_MULTIPLIER - 1) & categories) != 0)
|
||||
return -EINVAL;
|
||||
|
||||
acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results, num,
|
||||
categories, ctx->trans_table);
|
||||
|
||||
for (n = 0; n < MAX_SEARCHES_SCALAR; n++) {
|
||||
cmplt[n].count = 0;
|
||||
index_array[n] = acl_start_next_trie(&flows, parms, n, ctx);
|
||||
}
|
||||
|
||||
transition0 = index_array[0];
|
||||
transition1 = index_array[1];
|
||||
|
||||
while (flows.started > 0) {
|
||||
|
||||
input0 = GET_NEXT_4BYTES(parms, 0);
|
||||
input1 = GET_NEXT_4BYTES(parms, 1);
|
||||
|
||||
for (n = 0; n < 4; n++) {
|
||||
if (likely((transition0 & RTE_ACL_NODE_MATCH) == 0))
|
||||
transition0 = scalar_transition(flows.trans,
|
||||
transition0, (uint8_t)input0);
|
||||
|
||||
input0 >>= CHAR_BIT;
|
||||
|
||||
if (likely((transition1 & RTE_ACL_NODE_MATCH) == 0))
|
||||
transition1 = scalar_transition(flows.trans,
|
||||
transition1, (uint8_t)input1);
|
||||
|
||||
input1 >>= CHAR_BIT;
|
||||
|
||||
}
|
||||
if ((transition0 | transition1) & RTE_ACL_NODE_MATCH) {
|
||||
transition0 = acl_match_check(transition0,
|
||||
0, ctx, parms, &flows, resolve_priority_scalar);
|
||||
transition1 = acl_match_check(transition1,
|
||||
1, ctx, parms, &flows, resolve_priority_scalar);
|
||||
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
@ -31,24 +31,7 @@
|
||||
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
*/
|
||||
|
||||
#include <rte_acl.h>
|
||||
#include "acl_vect.h"
|
||||
#include "acl.h"
|
||||
|
||||
#define MAX_SEARCHES_SSE8 8
|
||||
#define MAX_SEARCHES_SSE4 4
|
||||
#define MAX_SEARCHES_SSE2 2
|
||||
#define MAX_SEARCHES_SCALAR 2
|
||||
|
||||
#define GET_NEXT_4BYTES(prm, idx) \
|
||||
(*((const int32_t *)((prm)[(idx)].data + *(prm)[idx].data_index++)))
|
||||
|
||||
|
||||
#define RTE_ACL_NODE_INDEX ((uint32_t)~RTE_ACL_NODE_TYPE)
|
||||
|
||||
#define SCALAR_QRANGE_MULT 0x01010101
|
||||
#define SCALAR_QRANGE_MASK 0x7f7f7f7f
|
||||
#define SCALAR_QRANGE_MIN 0x80808080
|
||||
#include "acl_run.h"
|
||||
|
||||
enum {
|
||||
SHUFFLE32_SLOT1 = 0xe5,
|
||||
@ -57,60 +40,6 @@ enum {
|
||||
SHUFFLE32_SWAP64 = 0x4e,
|
||||
};
|
||||
|
||||
/*
|
||||
* Structure to manage N parallel trie traversals.
|
||||
* The runtime trie traversal routines can process 8, 4, or 2 tries
|
||||
* in parallel. Each packet may require multiple trie traversals (up to 4).
|
||||
* This structure is used to fill the slots (0 to n-1) for parallel processing
|
||||
* with the trie traversals needed for each packet.
|
||||
*/
|
||||
struct acl_flow_data {
|
||||
uint32_t num_packets;
|
||||
/* number of packets processed */
|
||||
uint32_t started;
|
||||
/* number of trie traversals in progress */
|
||||
uint32_t trie;
|
||||
/* current trie index (0 to N-1) */
|
||||
uint32_t cmplt_size;
|
||||
uint32_t total_packets;
|
||||
uint32_t categories;
|
||||
/* number of result categories per packet. */
|
||||
/* maximum number of packets to process */
|
||||
const uint64_t *trans;
|
||||
const uint8_t **data;
|
||||
uint32_t *results;
|
||||
struct completion *last_cmplt;
|
||||
struct completion *cmplt_array;
|
||||
};
|
||||
|
||||
/*
|
||||
* Structure to maintain running results for
|
||||
* a single packet (up to 4 tries).
|
||||
*/
|
||||
struct completion {
|
||||
uint32_t *results; /* running results. */
|
||||
int32_t priority[RTE_ACL_MAX_CATEGORIES]; /* running priorities. */
|
||||
uint32_t count; /* num of remaining tries */
|
||||
/* true for allocated struct */
|
||||
} __attribute__((aligned(XMM_SIZE)));
|
||||
|
||||
/*
|
||||
* One parms structure for each slot in the search engine.
|
||||
*/
|
||||
struct parms {
|
||||
const uint8_t *data;
|
||||
/* input data for this packet */
|
||||
const uint32_t *data_index;
|
||||
/* data indirection for this trie */
|
||||
struct completion *cmplt;
|
||||
/* completion data for this packet */
|
||||
};
|
||||
|
||||
/*
|
||||
* Define an global idle node for unused engine slots
|
||||
*/
|
||||
static const uint32_t idle[UINT8_MAX + 1];
|
||||
|
||||
static const rte_xmm_t mm_type_quad_range = {
|
||||
.u32 = {
|
||||
RTE_ACL_NODE_QRANGE,
|
||||
@ -185,57 +114,16 @@ static const rte_xmm_t mm_index_mask64 = {
|
||||
},
|
||||
};
|
||||
|
||||
/*
|
||||
* Allocate a completion structure to manage the tries for a packet.
|
||||
*/
|
||||
static inline struct completion *
|
||||
alloc_completion(struct completion *p, uint32_t size, uint32_t tries,
|
||||
uint32_t *results)
|
||||
{
|
||||
uint32_t n;
|
||||
|
||||
for (n = 0; n < size; n++) {
|
||||
|
||||
if (p[n].count == 0) {
|
||||
|
||||
/* mark as allocated and set number of tries. */
|
||||
p[n].count = tries;
|
||||
p[n].results = results;
|
||||
return &(p[n]);
|
||||
}
|
||||
}
|
||||
|
||||
/* should never get here */
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/*
|
||||
* Resolve priority for a single result trie.
|
||||
* Resolve priority for multiple results (sse version).
|
||||
* This consists comparing the priority of the current traversal with the
|
||||
* running set of results for the packet.
|
||||
* For each result, keep a running array of the result (rule number) and
|
||||
* its priority for each category.
|
||||
*/
|
||||
static inline void
|
||||
resolve_single_priority(uint64_t transition, int n,
|
||||
const struct rte_acl_ctx *ctx, struct parms *parms,
|
||||
const struct rte_acl_match_results *p)
|
||||
{
|
||||
if (parms[n].cmplt->count == ctx->num_tries ||
|
||||
parms[n].cmplt->priority[0] <=
|
||||
p[transition].priority[0]) {
|
||||
|
||||
parms[n].cmplt->priority[0] = p[transition].priority[0];
|
||||
parms[n].cmplt->results[0] = p[transition].results[0];
|
||||
}
|
||||
|
||||
parms[n].cmplt->count--;
|
||||
}
|
||||
|
||||
/*
|
||||
* Resolve priority for multiple results. This consists comparing
|
||||
* the priority of the current traversal with the running set of
|
||||
* results for the packet. For each result, keep a running array of
|
||||
* the result (rule number) and its priority for each category.
|
||||
*/
|
||||
static inline void
|
||||
resolve_priority(uint64_t transition, int n, const struct rte_acl_ctx *ctx,
|
||||
resolve_priority_sse(uint64_t transition, int n, const struct rte_acl_ctx *ctx,
|
||||
struct parms *parms, const struct rte_acl_match_results *p,
|
||||
uint32_t categories)
|
||||
{
|
||||
@ -270,100 +158,6 @@ resolve_priority(uint64_t transition, int n, const struct rte_acl_ctx *ctx,
|
||||
MM_STOREU(saved_results, results);
|
||||
MM_STOREU(saved_priority, priority);
|
||||
}
|
||||
|
||||
/* Count down completed tries for this search request */
|
||||
parms[n].cmplt->count--;
|
||||
}
|
||||
|
||||
/*
|
||||
* Routine to fill a slot in the parallel trie traversal array (parms) from
|
||||
* the list of packets (flows).
|
||||
*/
|
||||
static inline uint64_t
|
||||
acl_start_next_trie(struct acl_flow_data *flows, struct parms *parms, int n,
|
||||
const struct rte_acl_ctx *ctx)
|
||||
{
|
||||
uint64_t transition;
|
||||
|
||||
/* if there are any more packets to process */
|
||||
if (flows->num_packets < flows->total_packets) {
|
||||
parms[n].data = flows->data[flows->num_packets];
|
||||
parms[n].data_index = ctx->trie[flows->trie].data_index;
|
||||
|
||||
/* if this is the first trie for this packet */
|
||||
if (flows->trie == 0) {
|
||||
flows->last_cmplt = alloc_completion(flows->cmplt_array,
|
||||
flows->cmplt_size, ctx->num_tries,
|
||||
flows->results +
|
||||
flows->num_packets * flows->categories);
|
||||
}
|
||||
|
||||
/* set completion parameters and starting index for this slot */
|
||||
parms[n].cmplt = flows->last_cmplt;
|
||||
transition =
|
||||
flows->trans[parms[n].data[*parms[n].data_index++] +
|
||||
ctx->trie[flows->trie].root_index];
|
||||
|
||||
/*
|
||||
* if this is the last trie for this packet,
|
||||
* then setup next packet.
|
||||
*/
|
||||
flows->trie++;
|
||||
if (flows->trie >= ctx->num_tries) {
|
||||
flows->trie = 0;
|
||||
flows->num_packets++;
|
||||
}
|
||||
|
||||
/* keep track of number of active trie traversals */
|
||||
flows->started++;
|
||||
|
||||
/* no more tries to process, set slot to an idle position */
|
||||
} else {
|
||||
transition = ctx->idle;
|
||||
parms[n].data = (const uint8_t *)idle;
|
||||
parms[n].data_index = idle;
|
||||
}
|
||||
return transition;
|
||||
}
|
||||
|
||||
/*
|
||||
* Detect matches. If a match node transition is found, then this trie
|
||||
* traversal is complete and fill the slot with the next trie
|
||||
* to be processed.
|
||||
*/
|
||||
static inline uint64_t
|
||||
acl_match_check_transition(uint64_t transition, int slot,
|
||||
const struct rte_acl_ctx *ctx, struct parms *parms,
|
||||
struct acl_flow_data *flows)
|
||||
{
|
||||
const struct rte_acl_match_results *p;
|
||||
|
||||
p = (const struct rte_acl_match_results *)
|
||||
(flows->trans + ctx->match_index);
|
||||
|
||||
if (transition & RTE_ACL_NODE_MATCH) {
|
||||
|
||||
/* Remove flags from index and decrement active traversals */
|
||||
transition &= RTE_ACL_NODE_INDEX;
|
||||
flows->started--;
|
||||
|
||||
/* Resolve priorities for this trie and running results */
|
||||
if (flows->categories == 1)
|
||||
resolve_single_priority(transition, slot, ctx,
|
||||
parms, p);
|
||||
else
|
||||
resolve_priority(transition, slot, ctx, parms, p,
|
||||
flows->categories);
|
||||
|
||||
/* Fill the slot with the next trie or idle trie */
|
||||
transition = acl_start_next_trie(flows, parms, slot, ctx);
|
||||
|
||||
} else if (transition == ctx->idle) {
|
||||
/* reset indirection table for idle slots */
|
||||
parms[slot].data_index = idle;
|
||||
}
|
||||
|
||||
return transition;
|
||||
}
|
||||
|
||||
/*
|
||||
@ -382,10 +176,10 @@ acl_process_matches(xmm_t *indicies, int slot, const struct rte_acl_ctx *ctx,
|
||||
*indicies = MM_SHUFFLE32(*indicies, SHUFFLE32_SWAP64);
|
||||
transition2 = MM_CVT64(*indicies);
|
||||
|
||||
transition1 = acl_match_check_transition(transition1, slot, ctx,
|
||||
parms, flows);
|
||||
transition2 = acl_match_check_transition(transition2, slot + 1, ctx,
|
||||
parms, flows);
|
||||
transition1 = acl_match_check(transition1, slot, ctx,
|
||||
parms, flows, resolve_priority_sse);
|
||||
transition2 = acl_match_check(transition2, slot + 1, ctx,
|
||||
parms, flows, resolve_priority_sse);
|
||||
|
||||
/* update indicies with new transitions. */
|
||||
*indicies = MM_SET64(transition2, transition1);
|
||||
@ -551,28 +345,10 @@ transition4(xmm_t index_mask, xmm_t next_input, xmm_t shuffle_input,
|
||||
return MM_SRL32(next_input, 8);
|
||||
}
|
||||
|
||||
static inline void
|
||||
acl_set_flow(struct acl_flow_data *flows, struct completion *cmplt,
|
||||
uint32_t cmplt_size, const uint8_t **data, uint32_t *results,
|
||||
uint32_t data_num, uint32_t categories, const uint64_t *trans)
|
||||
{
|
||||
flows->num_packets = 0;
|
||||
flows->started = 0;
|
||||
flows->trie = 0;
|
||||
flows->last_cmplt = NULL;
|
||||
flows->cmplt_array = cmplt;
|
||||
flows->total_packets = data_num;
|
||||
flows->categories = categories;
|
||||
flows->cmplt_size = cmplt_size;
|
||||
flows->data = data;
|
||||
flows->results = results;
|
||||
flows->trans = trans;
|
||||
}
|
||||
|
||||
/*
|
||||
* Execute trie traversal with 8 traversals in parallel
|
||||
*/
|
||||
static inline void
|
||||
static inline int
|
||||
search_sse_8(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t total_packets, uint32_t categories)
|
||||
{
|
||||
@ -676,12 +452,14 @@ search_sse_8(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
acl_match_check_x4(4, ctx, parms, &flows,
|
||||
&indicies3, &indicies4, mm_match_mask.m);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Execute trie traversal with 4 traversals in parallel
|
||||
*/
|
||||
static inline void
|
||||
static inline int
|
||||
search_sse_4(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, int total_packets, uint32_t categories)
|
||||
{
|
||||
@ -740,6 +518,8 @@ search_sse_4(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
acl_match_check_x4(0, ctx, parms, &flows,
|
||||
&indicies1, &indicies2, mm_match_mask.m);
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline xmm_t
|
||||
@ -769,7 +549,7 @@ transition2(xmm_t index_mask, xmm_t next_input, xmm_t shuffle_input,
|
||||
/*
|
||||
* Execute trie traversal with 2 traversals in parallel.
|
||||
*/
|
||||
static inline void
|
||||
static inline int
|
||||
search_sse_2(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t total_packets, uint32_t categories)
|
||||
{
|
||||
@ -825,108 +605,12 @@ search_sse_2(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
acl_match_check_x2(0, ctx, parms, &flows, &indicies,
|
||||
mm_match_mask64.m);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* When processing the transition, rather than using if/else
|
||||
* construct, the offset is calculated for DFA and QRANGE and
|
||||
* then conditionally added to the address based on node type.
|
||||
* This is done to avoid branch mis-predictions. Since the
|
||||
* offset is rather simple calculation it is more efficient
|
||||
* to do the calculation and do a condition move rather than
|
||||
* a conditional branch to determine which calculation to do.
|
||||
*/
|
||||
static inline uint32_t
|
||||
scan_forward(uint32_t input, uint32_t max)
|
||||
{
|
||||
return (input == 0) ? max : rte_bsf32(input);
|
||||
}
|
||||
|
||||
static inline uint64_t
|
||||
scalar_transition(const uint64_t *trans_table, uint64_t transition,
|
||||
uint8_t input)
|
||||
{
|
||||
uint32_t addr, index, ranges, x, a, b, c;
|
||||
|
||||
/* break transition into component parts */
|
||||
ranges = transition >> (sizeof(index) * CHAR_BIT);
|
||||
|
||||
/* calc address for a QRANGE node */
|
||||
c = input * SCALAR_QRANGE_MULT;
|
||||
a = ranges | SCALAR_QRANGE_MIN;
|
||||
index = transition & ~RTE_ACL_NODE_INDEX;
|
||||
a -= (c & SCALAR_QRANGE_MASK);
|
||||
b = c & SCALAR_QRANGE_MIN;
|
||||
addr = transition ^ index;
|
||||
a &= SCALAR_QRANGE_MIN;
|
||||
a ^= (ranges ^ b) & (a ^ b);
|
||||
x = scan_forward(a, 32) >> 3;
|
||||
addr += (index == RTE_ACL_NODE_DFA) ? input : x;
|
||||
|
||||
/* pickup next transition */
|
||||
transition = *(trans_table + addr);
|
||||
return transition;
|
||||
}
|
||||
|
||||
int
|
||||
rte_acl_classify_scalar(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories)
|
||||
{
|
||||
int n;
|
||||
uint64_t transition0, transition1;
|
||||
uint32_t input0, input1;
|
||||
struct acl_flow_data flows;
|
||||
uint64_t index_array[MAX_SEARCHES_SCALAR];
|
||||
struct completion cmplt[MAX_SEARCHES_SCALAR];
|
||||
struct parms parms[MAX_SEARCHES_SCALAR];
|
||||
|
||||
if (categories != 1 &&
|
||||
((RTE_ACL_RESULTS_MULTIPLIER - 1) & categories) != 0)
|
||||
return -EINVAL;
|
||||
|
||||
acl_set_flow(&flows, cmplt, RTE_DIM(cmplt), data, results, num,
|
||||
categories, ctx->trans_table);
|
||||
|
||||
for (n = 0; n < MAX_SEARCHES_SCALAR; n++) {
|
||||
cmplt[n].count = 0;
|
||||
index_array[n] = acl_start_next_trie(&flows, parms, n, ctx);
|
||||
}
|
||||
|
||||
transition0 = index_array[0];
|
||||
transition1 = index_array[1];
|
||||
|
||||
while (flows.started > 0) {
|
||||
|
||||
input0 = GET_NEXT_4BYTES(parms, 0);
|
||||
input1 = GET_NEXT_4BYTES(parms, 1);
|
||||
|
||||
for (n = 0; n < 4; n++) {
|
||||
if (likely((transition0 & RTE_ACL_NODE_MATCH) == 0))
|
||||
transition0 = scalar_transition(flows.trans,
|
||||
transition0, (uint8_t)input0);
|
||||
|
||||
input0 >>= CHAR_BIT;
|
||||
|
||||
if (likely((transition1 & RTE_ACL_NODE_MATCH) == 0))
|
||||
transition1 = scalar_transition(flows.trans,
|
||||
transition1, (uint8_t)input1);
|
||||
|
||||
input1 >>= CHAR_BIT;
|
||||
|
||||
}
|
||||
if ((transition0 | transition1) & RTE_ACL_NODE_MATCH) {
|
||||
transition0 = acl_match_check_transition(transition0,
|
||||
0, ctx, parms, &flows);
|
||||
transition1 = acl_match_check_transition(transition1,
|
||||
1, ctx, parms, &flows);
|
||||
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int
|
||||
rte_acl_classify(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
rte_acl_classify_sse(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories)
|
||||
{
|
||||
if (categories != 1 &&
|
||||
@ -934,11 +618,9 @@ rte_acl_classify(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
return -EINVAL;
|
||||
|
||||
if (likely(num >= MAX_SEARCHES_SSE8))
|
||||
search_sse_8(ctx, data, results, num, categories);
|
||||
return search_sse_8(ctx, data, results, num, categories);
|
||||
else if (num >= MAX_SEARCHES_SSE4)
|
||||
search_sse_4(ctx, data, results, num, categories);
|
||||
return search_sse_4(ctx, data, results, num, categories);
|
||||
else
|
||||
search_sse_2(ctx, data, results, num, categories);
|
||||
|
||||
return 0;
|
||||
return search_sse_2(ctx, data, results, num, categories);
|
||||
}
|
||||
@ -38,6 +38,58 @@
|
||||
|
||||
TAILQ_HEAD(rte_acl_list, rte_tailq_entry);
|
||||
|
||||
static const rte_acl_classify_t classify_fns[] = {
|
||||
[RTE_ACL_CLASSIFY_DEFAULT] = rte_acl_classify_scalar,
|
||||
[RTE_ACL_CLASSIFY_SCALAR] = rte_acl_classify_scalar,
|
||||
[RTE_ACL_CLASSIFY_SSE] = rte_acl_classify_sse,
|
||||
};
|
||||
|
||||
/* by default, use always avaialbe scalar code path. */
|
||||
static enum rte_acl_classify_alg rte_acl_default_classify =
|
||||
RTE_ACL_CLASSIFY_SCALAR;
|
||||
|
||||
static void
|
||||
rte_acl_set_default_classify(enum rte_acl_classify_alg alg)
|
||||
{
|
||||
rte_acl_default_classify = alg;
|
||||
}
|
||||
|
||||
extern int
|
||||
rte_acl_set_ctx_classify(struct rte_acl_ctx *ctx, enum rte_acl_classify_alg alg)
|
||||
{
|
||||
if (ctx == NULL || (uint32_t)alg >= RTE_DIM(classify_fns))
|
||||
return -EINVAL;
|
||||
|
||||
ctx->alg = alg;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __attribute__((constructor))
|
||||
rte_acl_init(void)
|
||||
{
|
||||
enum rte_acl_classify_alg alg = RTE_ACL_CLASSIFY_DEFAULT;
|
||||
|
||||
if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_SSE4_1))
|
||||
alg = RTE_ACL_CLASSIFY_SSE;
|
||||
|
||||
rte_acl_set_default_classify(alg);
|
||||
}
|
||||
|
||||
int
|
||||
rte_acl_classify(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories)
|
||||
{
|
||||
return classify_fns[ctx->alg](ctx, data, results, num, categories);
|
||||
}
|
||||
|
||||
int
|
||||
rte_acl_classify_alg(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories,
|
||||
enum rte_acl_classify_alg alg)
|
||||
{
|
||||
return classify_fns[alg](ctx, data, results, num, categories);
|
||||
}
|
||||
|
||||
struct rte_acl_ctx *
|
||||
rte_acl_find_existing(const char *name)
|
||||
{
|
||||
@ -165,6 +217,7 @@ rte_acl_create(const struct rte_acl_param *param)
|
||||
ctx->max_rules = param->max_rule_num;
|
||||
ctx->rule_sz = param->rule_size;
|
||||
ctx->socket_id = param->socket_id;
|
||||
ctx->alg = rte_acl_default_classify;
|
||||
snprintf(ctx->name, sizeof(ctx->name), "%s", param->name);
|
||||
|
||||
te->data = (void *) ctx;
|
||||
@ -261,6 +314,8 @@ rte_acl_dump(const struct rte_acl_ctx *ctx)
|
||||
if (!ctx)
|
||||
return;
|
||||
printf("acl context <%s>@%p\n", ctx->name, ctx);
|
||||
printf(" socket_id=%"PRId32"\n", ctx->socket_id);
|
||||
printf(" alg=%"PRId32"\n", ctx->alg);
|
||||
printf(" max_rules=%"PRIu32"\n", ctx->max_rules);
|
||||
printf(" rule_size=%"PRIu32"\n", ctx->rule_sz);
|
||||
printf(" num_rules=%"PRIu32"\n", ctx->num_rules);
|
||||
|
||||
@ -259,43 +259,16 @@ void
|
||||
rte_acl_reset(struct rte_acl_ctx *ctx);
|
||||
|
||||
/**
|
||||
* Search for a matching ACL rule for each input data buffer.
|
||||
* Each input data buffer can have up to *categories* matches.
|
||||
* That implies that results array should be big enough to hold
|
||||
* (categories * num) elements.
|
||||
* Also categories parameter should be either one or multiple of
|
||||
* RTE_ACL_RESULTS_MULTIPLIER and can't be bigger than RTE_ACL_MAX_CATEGORIES.
|
||||
* If more than one rule is applicable for given input buffer and
|
||||
* given category, then rule with highest priority will be returned as a match.
|
||||
* Note, that it is a caller responsibility to ensure that input parameters
|
||||
* are valid and point to correct memory locations.
|
||||
*
|
||||
* @param ctx
|
||||
* ACL context to search with.
|
||||
* @param data
|
||||
* Array of pointers to input data buffers to perform search.
|
||||
* Note that all fields in input data buffers supposed to be in network
|
||||
* byte order (MSB).
|
||||
* @param results
|
||||
* Array of search results, *categories* results per each input data buffer.
|
||||
* @param num
|
||||
* Number of elements in the input data buffers array.
|
||||
* @param categories
|
||||
* Number of maximum possible matches for each input buffer, one possible
|
||||
* match per category.
|
||||
* @return
|
||||
* zero on successful completion.
|
||||
* -EINVAL for incorrect arguments.
|
||||
* Avaialble implementations of ACL classify.
|
||||
*/
|
||||
int
|
||||
rte_acl_classify(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories);
|
||||
enum rte_acl_classify_alg {
|
||||
RTE_ACL_CLASSIFY_DEFAULT = 0,
|
||||
RTE_ACL_CLASSIFY_SCALAR = 1, /**< generic implementation. */
|
||||
RTE_ACL_CLASSIFY_SSE = 2, /**< requries SSE4.1 support. */
|
||||
};
|
||||
|
||||
/**
|
||||
* Perform scalar search for a matching ACL rule for each input data buffer.
|
||||
* Note, that while the search itself will avoid explicit use of SSE/AVX
|
||||
* intrinsics, code for comparing matching results/priorities sill might use
|
||||
* vector intrinsics (for categories > 1).
|
||||
* Perform search for a matching ACL rule for each input data buffer.
|
||||
* Each input data buffer can have up to *categories* matches.
|
||||
* That implies that results array should be big enough to hold
|
||||
* (categories * num) elements.
|
||||
@ -323,9 +296,68 @@ rte_acl_classify(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
* zero on successful completion.
|
||||
* -EINVAL for incorrect arguments.
|
||||
*/
|
||||
int
|
||||
rte_acl_classify_scalar(const struct rte_acl_ctx *ctx, const uint8_t **data,
|
||||
uint32_t *results, uint32_t num, uint32_t categories);
|
||||
extern int
|
||||
rte_acl_classify(const struct rte_acl_ctx *ctx,
|
||||
const uint8_t **data,
|
||||
uint32_t *results, uint32_t num,
|
||||
uint32_t categories);
|
||||
|
||||
/**
|
||||
* Perform search using specified algorithm for a matching ACL rule for
|
||||
* each input data buffer.
|
||||
* Each input data buffer can have up to *categories* matches.
|
||||
* That implies that results array should be big enough to hold
|
||||
* (categories * num) elements.
|
||||
* Also categories parameter should be either one or multiple of
|
||||
* RTE_ACL_RESULTS_MULTIPLIER and can't be bigger than RTE_ACL_MAX_CATEGORIES.
|
||||
* If more than one rule is applicable for given input buffer and
|
||||
* given category, then rule with highest priority will be returned as a match.
|
||||
* Note, that it is a caller's responsibility to ensure that input parameters
|
||||
* are valid and point to correct memory locations.
|
||||
*
|
||||
* @param ctx
|
||||
* ACL context to search with.
|
||||
* @param data
|
||||
* Array of pointers to input data buffers to perform search.
|
||||
* Note that all fields in input data buffers supposed to be in network
|
||||
* byte order (MSB).
|
||||
* @param results
|
||||
* Array of search results, *categories* results per each input data buffer.
|
||||
* @param num
|
||||
* Number of elements in the input data buffers array.
|
||||
* @param categories
|
||||
* Number of maximum possible matches for each input buffer, one possible
|
||||
* match per category.
|
||||
* @param alg
|
||||
* Algorithm to be used for the search.
|
||||
* It is the caller responibility to ensure that the value refers to the
|
||||
* existing algorithm, and that it could be run on the given CPU.
|
||||
* @return
|
||||
* zero on successful completion.
|
||||
* -EINVAL for incorrect arguments.
|
||||
*/
|
||||
extern int
|
||||
rte_acl_classify_alg(const struct rte_acl_ctx *ctx,
|
||||
const uint8_t **data,
|
||||
uint32_t *results, uint32_t num,
|
||||
uint32_t categories,
|
||||
enum rte_acl_classify_alg alg);
|
||||
|
||||
/*
|
||||
* Override the default classifier function for a given ACL context.
|
||||
* @param ctx
|
||||
* ACL context to change classify function for.
|
||||
* @param alg
|
||||
* New default classify algorithm for given ACL context.
|
||||
* It is the caller responibility to ensure that the value refers to the
|
||||
* existing algorithm, and that it could be run on the given CPU.
|
||||
* @return
|
||||
* - -EINVAL if the parameters are invalid.
|
||||
* - Zero if operation completed successfully.
|
||||
*/
|
||||
extern int
|
||||
rte_acl_set_ctx_classify(struct rte_acl_ctx *ctx,
|
||||
enum rte_acl_classify_alg alg);
|
||||
|
||||
/**
|
||||
* Dump an ACL context structure to the console.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user