175d213bf8
The arguments of actions that are learned are now specified as part of the learn instruction as opposed to being statically specified as part of the learner table configuration. Signed-off-by: Cristian Dumitrescu <cristian.dumitrescu@intel.com>
3895 lines
102 KiB
C
3895 lines
102 KiB
C
/* SPDX-License-Identifier: BSD-3-Clause
|
|
* Copyright(c) 2021 Intel Corporation
|
|
*/
|
|
#ifndef __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
|
|
#define __INCLUDE_RTE_SWX_PIPELINE_INTERNAL_H__
|
|
|
|
#include <inttypes.h>
|
|
#include <string.h>
|
|
#include <sys/queue.h>
|
|
|
|
#include <rte_byteorder.h>
|
|
#include <rte_common.h>
|
|
#include <rte_cycles.h>
|
|
#include <rte_prefetch.h>
|
|
#include <rte_meter.h>
|
|
|
|
#include <rte_swx_table_selector.h>
|
|
#include <rte_swx_table_learner.h>
|
|
#include <rte_swx_pipeline.h>
|
|
#include <rte_swx_ctl.h>
|
|
|
|
#ifndef TRACE_LEVEL
|
|
#define TRACE_LEVEL 0
|
|
#endif
|
|
|
|
#if TRACE_LEVEL
|
|
#define TRACE(...) printf(__VA_ARGS__)
|
|
#else
|
|
#define TRACE(...)
|
|
#endif
|
|
|
|
/*
|
|
* Environment.
|
|
*/
|
|
#define ntoh64(x) rte_be_to_cpu_64(x)
|
|
#define hton64(x) rte_cpu_to_be_64(x)
|
|
|
|
/*
|
|
* Struct.
|
|
*/
|
|
struct field {
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
uint32_t n_bits;
|
|
uint32_t offset;
|
|
int var_size;
|
|
};
|
|
|
|
struct struct_type {
|
|
TAILQ_ENTRY(struct_type) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct field *fields;
|
|
uint32_t n_fields;
|
|
uint32_t n_bits;
|
|
uint32_t n_bits_min;
|
|
int var_size;
|
|
};
|
|
|
|
TAILQ_HEAD(struct_type_tailq, struct_type);
|
|
|
|
/*
|
|
* Input port.
|
|
*/
|
|
struct port_in_type {
|
|
TAILQ_ENTRY(port_in_type) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct rte_swx_port_in_ops ops;
|
|
};
|
|
|
|
TAILQ_HEAD(port_in_type_tailq, port_in_type);
|
|
|
|
struct port_in {
|
|
TAILQ_ENTRY(port_in) node;
|
|
struct port_in_type *type;
|
|
void *obj;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(port_in_tailq, port_in);
|
|
|
|
struct port_in_runtime {
|
|
rte_swx_port_in_pkt_rx_t pkt_rx;
|
|
void *obj;
|
|
};
|
|
|
|
/*
|
|
* Output port.
|
|
*/
|
|
struct port_out_type {
|
|
TAILQ_ENTRY(port_out_type) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct rte_swx_port_out_ops ops;
|
|
};
|
|
|
|
TAILQ_HEAD(port_out_type_tailq, port_out_type);
|
|
|
|
struct port_out {
|
|
TAILQ_ENTRY(port_out) node;
|
|
struct port_out_type *type;
|
|
void *obj;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(port_out_tailq, port_out);
|
|
|
|
struct port_out_runtime {
|
|
rte_swx_port_out_pkt_tx_t pkt_tx;
|
|
rte_swx_port_out_flush_t flush;
|
|
void *obj;
|
|
};
|
|
|
|
/*
|
|
* Extern object.
|
|
*/
|
|
struct extern_type_member_func {
|
|
TAILQ_ENTRY(extern_type_member_func) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
rte_swx_extern_type_member_func_t func;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(extern_type_member_func_tailq, extern_type_member_func);
|
|
|
|
struct extern_type {
|
|
TAILQ_ENTRY(extern_type) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct struct_type *mailbox_struct_type;
|
|
rte_swx_extern_type_constructor_t constructor;
|
|
rte_swx_extern_type_destructor_t destructor;
|
|
struct extern_type_member_func_tailq funcs;
|
|
uint32_t n_funcs;
|
|
};
|
|
|
|
TAILQ_HEAD(extern_type_tailq, extern_type);
|
|
|
|
struct extern_obj {
|
|
TAILQ_ENTRY(extern_obj) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct extern_type *type;
|
|
void *obj;
|
|
uint32_t struct_id;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(extern_obj_tailq, extern_obj);
|
|
|
|
#ifndef RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX
|
|
#define RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX 8
|
|
#endif
|
|
|
|
struct extern_obj_runtime {
|
|
void *obj;
|
|
uint8_t *mailbox;
|
|
rte_swx_extern_type_member_func_t funcs[RTE_SWX_EXTERN_TYPE_MEMBER_FUNCS_MAX];
|
|
};
|
|
|
|
/*
|
|
* Extern function.
|
|
*/
|
|
struct extern_func {
|
|
TAILQ_ENTRY(extern_func) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct struct_type *mailbox_struct_type;
|
|
rte_swx_extern_func_t func;
|
|
uint32_t struct_id;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(extern_func_tailq, extern_func);
|
|
|
|
struct extern_func_runtime {
|
|
uint8_t *mailbox;
|
|
rte_swx_extern_func_t func;
|
|
};
|
|
|
|
/*
|
|
* Header.
|
|
*/
|
|
struct header {
|
|
TAILQ_ENTRY(header) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct struct_type *st;
|
|
uint32_t struct_id;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(header_tailq, header);
|
|
|
|
struct header_runtime {
|
|
uint8_t *ptr0;
|
|
uint32_t n_bytes;
|
|
};
|
|
|
|
struct header_out_runtime {
|
|
uint8_t *ptr0;
|
|
uint8_t *ptr;
|
|
uint32_t n_bytes;
|
|
};
|
|
|
|
/*
|
|
* Instruction.
|
|
*/
|
|
|
|
/* Packet headers are always in Network Byte Order (NBO), i.e. big endian.
|
|
* Packet meta-data fields are always assumed to be in Host Byte Order (HBO).
|
|
* Table entry fields can be in either NBO or HBO; they are assumed to be in HBO
|
|
* when transferred to packet meta-data and in NBO when transferred to packet
|
|
* headers.
|
|
*/
|
|
|
|
/* Notation conventions:
|
|
* -Header field: H = h.header.field (dst/src)
|
|
* -Meta-data field: M = m.field (dst/src)
|
|
* -Extern object mailbox field: E = e.field (dst/src)
|
|
* -Extern function mailbox field: F = f.field (dst/src)
|
|
* -Table action data field: T = t.field (src only)
|
|
* -Immediate value: I = 32-bit unsigned value (src only)
|
|
*/
|
|
|
|
enum instruction_type {
|
|
/* rx m.port_in */
|
|
INSTR_RX,
|
|
|
|
/* tx port_out
|
|
* port_out = MI
|
|
*/
|
|
INSTR_TX, /* port_out = M */
|
|
INSTR_TX_I, /* port_out = I */
|
|
|
|
/* extract h.header */
|
|
INSTR_HDR_EXTRACT,
|
|
INSTR_HDR_EXTRACT2,
|
|
INSTR_HDR_EXTRACT3,
|
|
INSTR_HDR_EXTRACT4,
|
|
INSTR_HDR_EXTRACT5,
|
|
INSTR_HDR_EXTRACT6,
|
|
INSTR_HDR_EXTRACT7,
|
|
INSTR_HDR_EXTRACT8,
|
|
|
|
/* extract h.header m.last_field_size */
|
|
INSTR_HDR_EXTRACT_M,
|
|
|
|
/* lookahead h.header */
|
|
INSTR_HDR_LOOKAHEAD,
|
|
|
|
/* emit h.header */
|
|
INSTR_HDR_EMIT,
|
|
INSTR_HDR_EMIT_TX,
|
|
INSTR_HDR_EMIT2_TX,
|
|
INSTR_HDR_EMIT3_TX,
|
|
INSTR_HDR_EMIT4_TX,
|
|
INSTR_HDR_EMIT5_TX,
|
|
INSTR_HDR_EMIT6_TX,
|
|
INSTR_HDR_EMIT7_TX,
|
|
INSTR_HDR_EMIT8_TX,
|
|
|
|
/* validate h.header */
|
|
INSTR_HDR_VALIDATE,
|
|
|
|
/* invalidate h.header */
|
|
INSTR_HDR_INVALIDATE,
|
|
|
|
/* mov dst src
|
|
* dst = src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_MOV, /* dst = MEF, src = MEFT */
|
|
INSTR_MOV_MH, /* dst = MEF, src = H */
|
|
INSTR_MOV_HM, /* dst = H, src = MEFT */
|
|
INSTR_MOV_HH, /* dst = H, src = H */
|
|
INSTR_MOV_I, /* dst = HMEF, src = I */
|
|
|
|
/* dma h.header t.field
|
|
* memcpy(h.header, t.field, sizeof(h.header))
|
|
*/
|
|
INSTR_DMA_HT,
|
|
INSTR_DMA_HT2,
|
|
INSTR_DMA_HT3,
|
|
INSTR_DMA_HT4,
|
|
INSTR_DMA_HT5,
|
|
INSTR_DMA_HT6,
|
|
INSTR_DMA_HT7,
|
|
INSTR_DMA_HT8,
|
|
|
|
/* add dst src
|
|
* dst += src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_ADD, /* dst = MEF, src = MEF */
|
|
INSTR_ALU_ADD_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_ADD_HM, /* dst = H, src = MEF */
|
|
INSTR_ALU_ADD_HH, /* dst = H, src = H */
|
|
INSTR_ALU_ADD_MI, /* dst = MEF, src = I */
|
|
INSTR_ALU_ADD_HI, /* dst = H, src = I */
|
|
|
|
/* sub dst src
|
|
* dst -= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_SUB, /* dst = MEF, src = MEF */
|
|
INSTR_ALU_SUB_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_SUB_HM, /* dst = H, src = MEF */
|
|
INSTR_ALU_SUB_HH, /* dst = H, src = H */
|
|
INSTR_ALU_SUB_MI, /* dst = MEF, src = I */
|
|
INSTR_ALU_SUB_HI, /* dst = H, src = I */
|
|
|
|
/* ckadd dst src
|
|
* dst = dst '+ src[0:1] '+ src[2:3] + ...
|
|
* dst = H, src = {H, h.header}
|
|
*/
|
|
INSTR_ALU_CKADD_FIELD, /* src = H */
|
|
INSTR_ALU_CKADD_STRUCT20, /* src = h.header, with sizeof(header) = 20 */
|
|
INSTR_ALU_CKADD_STRUCT, /* src = h.hdeader, with any sizeof(header) */
|
|
|
|
/* cksub dst src
|
|
* dst = dst '- src
|
|
* dst = H, src = H
|
|
*/
|
|
INSTR_ALU_CKSUB_FIELD,
|
|
|
|
/* and dst src
|
|
* dst &= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_AND, /* dst = MEF, src = MEFT */
|
|
INSTR_ALU_AND_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_AND_HM, /* dst = H, src = MEFT */
|
|
INSTR_ALU_AND_HH, /* dst = H, src = H */
|
|
INSTR_ALU_AND_I, /* dst = HMEF, src = I */
|
|
|
|
/* or dst src
|
|
* dst |= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_OR, /* dst = MEF, src = MEFT */
|
|
INSTR_ALU_OR_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_OR_HM, /* dst = H, src = MEFT */
|
|
INSTR_ALU_OR_HH, /* dst = H, src = H */
|
|
INSTR_ALU_OR_I, /* dst = HMEF, src = I */
|
|
|
|
/* xor dst src
|
|
* dst ^= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_XOR, /* dst = MEF, src = MEFT */
|
|
INSTR_ALU_XOR_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_XOR_HM, /* dst = H, src = MEFT */
|
|
INSTR_ALU_XOR_HH, /* dst = H, src = H */
|
|
INSTR_ALU_XOR_I, /* dst = HMEF, src = I */
|
|
|
|
/* shl dst src
|
|
* dst <<= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_SHL, /* dst = MEF, src = MEF */
|
|
INSTR_ALU_SHL_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_SHL_HM, /* dst = H, src = MEF */
|
|
INSTR_ALU_SHL_HH, /* dst = H, src = H */
|
|
INSTR_ALU_SHL_MI, /* dst = MEF, src = I */
|
|
INSTR_ALU_SHL_HI, /* dst = H, src = I */
|
|
|
|
/* shr dst src
|
|
* dst >>= src
|
|
* dst = HMEF, src = HMEFTI
|
|
*/
|
|
INSTR_ALU_SHR, /* dst = MEF, src = MEF */
|
|
INSTR_ALU_SHR_MH, /* dst = MEF, src = H */
|
|
INSTR_ALU_SHR_HM, /* dst = H, src = MEF */
|
|
INSTR_ALU_SHR_HH, /* dst = H, src = H */
|
|
INSTR_ALU_SHR_MI, /* dst = MEF, src = I */
|
|
INSTR_ALU_SHR_HI, /* dst = H, src = I */
|
|
|
|
/* regprefetch REGARRAY index
|
|
* prefetch REGARRAY[index]
|
|
* index = HMEFTI
|
|
*/
|
|
INSTR_REGPREFETCH_RH, /* index = H */
|
|
INSTR_REGPREFETCH_RM, /* index = MEFT */
|
|
INSTR_REGPREFETCH_RI, /* index = I */
|
|
|
|
/* regrd dst REGARRAY index
|
|
* dst = REGARRAY[index]
|
|
* dst = HMEF, index = HMEFTI
|
|
*/
|
|
INSTR_REGRD_HRH, /* dst = H, index = H */
|
|
INSTR_REGRD_HRM, /* dst = H, index = MEFT */
|
|
INSTR_REGRD_HRI, /* dst = H, index = I */
|
|
INSTR_REGRD_MRH, /* dst = MEF, index = H */
|
|
INSTR_REGRD_MRM, /* dst = MEF, index = MEFT */
|
|
INSTR_REGRD_MRI, /* dst = MEF, index = I */
|
|
|
|
/* regwr REGARRAY index src
|
|
* REGARRAY[index] = src
|
|
* index = HMEFTI, src = HMEFTI
|
|
*/
|
|
INSTR_REGWR_RHH, /* index = H, src = H */
|
|
INSTR_REGWR_RHM, /* index = H, src = MEFT */
|
|
INSTR_REGWR_RHI, /* index = H, src = I */
|
|
INSTR_REGWR_RMH, /* index = MEFT, src = H */
|
|
INSTR_REGWR_RMM, /* index = MEFT, src = MEFT */
|
|
INSTR_REGWR_RMI, /* index = MEFT, src = I */
|
|
INSTR_REGWR_RIH, /* index = I, src = H */
|
|
INSTR_REGWR_RIM, /* index = I, src = MEFT */
|
|
INSTR_REGWR_RII, /* index = I, src = I */
|
|
|
|
/* regadd REGARRAY index src
|
|
* REGARRAY[index] += src
|
|
* index = HMEFTI, src = HMEFTI
|
|
*/
|
|
INSTR_REGADD_RHH, /* index = H, src = H */
|
|
INSTR_REGADD_RHM, /* index = H, src = MEFT */
|
|
INSTR_REGADD_RHI, /* index = H, src = I */
|
|
INSTR_REGADD_RMH, /* index = MEFT, src = H */
|
|
INSTR_REGADD_RMM, /* index = MEFT, src = MEFT */
|
|
INSTR_REGADD_RMI, /* index = MEFT, src = I */
|
|
INSTR_REGADD_RIH, /* index = I, src = H */
|
|
INSTR_REGADD_RIM, /* index = I, src = MEFT */
|
|
INSTR_REGADD_RII, /* index = I, src = I */
|
|
|
|
/* metprefetch METARRAY index
|
|
* prefetch METARRAY[index]
|
|
* index = HMEFTI
|
|
*/
|
|
INSTR_METPREFETCH_H, /* index = H */
|
|
INSTR_METPREFETCH_M, /* index = MEFT */
|
|
INSTR_METPREFETCH_I, /* index = I */
|
|
|
|
/* meter METARRAY index length color_in color_out
|
|
* color_out = meter(METARRAY[index], length, color_in)
|
|
* index = HMEFTI, length = HMEFT, color_in = MEFTI, color_out = MEF
|
|
*/
|
|
INSTR_METER_HHM, /* index = H, length = H, color_in = MEFT */
|
|
INSTR_METER_HHI, /* index = H, length = H, color_in = I */
|
|
INSTR_METER_HMM, /* index = H, length = MEFT, color_in = MEFT */
|
|
INSTR_METER_HMI, /* index = H, length = MEFT, color_in = I */
|
|
INSTR_METER_MHM, /* index = MEFT, length = H, color_in = MEFT */
|
|
INSTR_METER_MHI, /* index = MEFT, length = H, color_in = I */
|
|
INSTR_METER_MMM, /* index = MEFT, length = MEFT, color_in = MEFT */
|
|
INSTR_METER_MMI, /* index = MEFT, length = MEFT, color_in = I */
|
|
INSTR_METER_IHM, /* index = I, length = H, color_in = MEFT */
|
|
INSTR_METER_IHI, /* index = I, length = H, color_in = I */
|
|
INSTR_METER_IMM, /* index = I, length = MEFT, color_in = MEFT */
|
|
INSTR_METER_IMI, /* index = I, length = MEFT, color_in = I */
|
|
|
|
/* table TABLE */
|
|
INSTR_TABLE,
|
|
INSTR_TABLE_AF,
|
|
INSTR_SELECTOR,
|
|
INSTR_LEARNER,
|
|
INSTR_LEARNER_AF,
|
|
|
|
/* learn LEARNER ACTION_NAME [ m.action_first_arg ] */
|
|
INSTR_LEARNER_LEARN,
|
|
|
|
/* forget */
|
|
INSTR_LEARNER_FORGET,
|
|
|
|
/* extern e.obj.func */
|
|
INSTR_EXTERN_OBJ,
|
|
|
|
/* extern f.func */
|
|
INSTR_EXTERN_FUNC,
|
|
|
|
/* jmp LABEL
|
|
* Unconditional jump
|
|
*/
|
|
INSTR_JMP,
|
|
|
|
/* jmpv LABEL h.header
|
|
* Jump if header is valid
|
|
*/
|
|
INSTR_JMP_VALID,
|
|
|
|
/* jmpnv LABEL h.header
|
|
* Jump if header is invalid
|
|
*/
|
|
INSTR_JMP_INVALID,
|
|
|
|
/* jmph LABEL
|
|
* Jump if table lookup hit
|
|
*/
|
|
INSTR_JMP_HIT,
|
|
|
|
/* jmpnh LABEL
|
|
* Jump if table lookup miss
|
|
*/
|
|
INSTR_JMP_MISS,
|
|
|
|
/* jmpa LABEL ACTION
|
|
* Jump if action run
|
|
*/
|
|
INSTR_JMP_ACTION_HIT,
|
|
|
|
/* jmpna LABEL ACTION
|
|
* Jump if action not run
|
|
*/
|
|
INSTR_JMP_ACTION_MISS,
|
|
|
|
/* jmpeq LABEL a b
|
|
* Jump if a is equal to b
|
|
* a = HMEFT, b = HMEFTI
|
|
*/
|
|
INSTR_JMP_EQ, /* a = MEFT, b = MEFT */
|
|
INSTR_JMP_EQ_MH, /* a = MEFT, b = H */
|
|
INSTR_JMP_EQ_HM, /* a = H, b = MEFT */
|
|
INSTR_JMP_EQ_HH, /* a = H, b = H */
|
|
INSTR_JMP_EQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
|
|
|
|
/* jmpneq LABEL a b
|
|
* Jump if a is not equal to b
|
|
* a = HMEFT, b = HMEFTI
|
|
*/
|
|
INSTR_JMP_NEQ, /* a = MEFT, b = MEFT */
|
|
INSTR_JMP_NEQ_MH, /* a = MEFT, b = H */
|
|
INSTR_JMP_NEQ_HM, /* a = H, b = MEFT */
|
|
INSTR_JMP_NEQ_HH, /* a = H, b = H */
|
|
INSTR_JMP_NEQ_I, /* (a, b) = (MEFT, I) or (a, b) = (H, I) */
|
|
|
|
/* jmplt LABEL a b
|
|
* Jump if a is less than b
|
|
* a = HMEFT, b = HMEFTI
|
|
*/
|
|
INSTR_JMP_LT, /* a = MEFT, b = MEFT */
|
|
INSTR_JMP_LT_MH, /* a = MEFT, b = H */
|
|
INSTR_JMP_LT_HM, /* a = H, b = MEFT */
|
|
INSTR_JMP_LT_HH, /* a = H, b = H */
|
|
INSTR_JMP_LT_MI, /* a = MEFT, b = I */
|
|
INSTR_JMP_LT_HI, /* a = H, b = I */
|
|
|
|
/* jmpgt LABEL a b
|
|
* Jump if a is greater than b
|
|
* a = HMEFT, b = HMEFTI
|
|
*/
|
|
INSTR_JMP_GT, /* a = MEFT, b = MEFT */
|
|
INSTR_JMP_GT_MH, /* a = MEFT, b = H */
|
|
INSTR_JMP_GT_HM, /* a = H, b = MEFT */
|
|
INSTR_JMP_GT_HH, /* a = H, b = H */
|
|
INSTR_JMP_GT_MI, /* a = MEFT, b = I */
|
|
INSTR_JMP_GT_HI, /* a = H, b = I */
|
|
|
|
/* return
|
|
* Return from action
|
|
*/
|
|
INSTR_RETURN,
|
|
|
|
/* Start of custom instructions. */
|
|
INSTR_CUSTOM_0,
|
|
};
|
|
|
|
struct instr_operand {
|
|
uint8_t struct_id;
|
|
uint8_t n_bits;
|
|
uint8_t offset;
|
|
uint8_t pad;
|
|
};
|
|
|
|
struct instr_io {
|
|
struct {
|
|
union {
|
|
struct {
|
|
uint8_t offset;
|
|
uint8_t n_bits;
|
|
uint8_t pad[2];
|
|
};
|
|
|
|
uint32_t val;
|
|
};
|
|
} io;
|
|
|
|
struct {
|
|
uint8_t header_id[8];
|
|
uint8_t struct_id[8];
|
|
uint8_t n_bytes[8];
|
|
} hdr;
|
|
};
|
|
|
|
struct instr_hdr_validity {
|
|
uint8_t header_id;
|
|
};
|
|
|
|
struct instr_table {
|
|
uint8_t table_id;
|
|
};
|
|
|
|
struct instr_learn {
|
|
uint8_t action_id;
|
|
uint8_t mf_offset;
|
|
};
|
|
|
|
struct instr_extern_obj {
|
|
uint8_t ext_obj_id;
|
|
uint8_t func_id;
|
|
};
|
|
|
|
struct instr_extern_func {
|
|
uint8_t ext_func_id;
|
|
};
|
|
|
|
struct instr_dst_src {
|
|
struct instr_operand dst;
|
|
union {
|
|
struct instr_operand src;
|
|
uint64_t src_val;
|
|
};
|
|
};
|
|
|
|
struct instr_regarray {
|
|
uint8_t regarray_id;
|
|
uint8_t pad[3];
|
|
|
|
union {
|
|
struct instr_operand idx;
|
|
uint32_t idx_val;
|
|
};
|
|
|
|
union {
|
|
struct instr_operand dstsrc;
|
|
uint64_t dstsrc_val;
|
|
};
|
|
};
|
|
|
|
struct instr_meter {
|
|
uint8_t metarray_id;
|
|
uint8_t pad[3];
|
|
|
|
union {
|
|
struct instr_operand idx;
|
|
uint32_t idx_val;
|
|
};
|
|
|
|
struct instr_operand length;
|
|
|
|
union {
|
|
struct instr_operand color_in;
|
|
uint32_t color_in_val;
|
|
};
|
|
|
|
struct instr_operand color_out;
|
|
};
|
|
|
|
struct instr_dma {
|
|
struct {
|
|
uint8_t header_id[8];
|
|
uint8_t struct_id[8];
|
|
} dst;
|
|
|
|
struct {
|
|
uint8_t offset[8];
|
|
} src;
|
|
|
|
uint16_t n_bytes[8];
|
|
};
|
|
|
|
struct instr_jmp {
|
|
struct instruction *ip;
|
|
|
|
union {
|
|
struct instr_operand a;
|
|
uint8_t header_id;
|
|
uint8_t action_id;
|
|
};
|
|
|
|
union {
|
|
struct instr_operand b;
|
|
uint64_t b_val;
|
|
};
|
|
};
|
|
|
|
struct instruction {
|
|
enum instruction_type type;
|
|
union {
|
|
struct instr_io io;
|
|
struct instr_hdr_validity valid;
|
|
struct instr_dst_src mov;
|
|
struct instr_regarray regarray;
|
|
struct instr_meter meter;
|
|
struct instr_dma dma;
|
|
struct instr_dst_src alu;
|
|
struct instr_table table;
|
|
struct instr_learn learn;
|
|
struct instr_extern_obj ext_obj;
|
|
struct instr_extern_func ext_func;
|
|
struct instr_jmp jmp;
|
|
};
|
|
};
|
|
|
|
struct instruction_data {
|
|
char label[RTE_SWX_NAME_SIZE];
|
|
char jmp_label[RTE_SWX_NAME_SIZE];
|
|
uint32_t n_users; /* user = jmp instruction to this instruction. */
|
|
int invalid;
|
|
};
|
|
|
|
typedef void (*instr_exec_t)(struct rte_swx_pipeline *);
|
|
|
|
/*
|
|
* Action.
|
|
*/
|
|
typedef void
|
|
(*action_func_t)(struct rte_swx_pipeline *p);
|
|
|
|
struct action {
|
|
TAILQ_ENTRY(action) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct struct_type *st;
|
|
int *args_endianness; /* 0 = Host Byte Order (HBO); 1 = Network Byte Order (NBO). */
|
|
struct instruction *instructions;
|
|
struct instruction_data *instruction_data;
|
|
uint32_t n_instructions;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(action_tailq, action);
|
|
|
|
/*
|
|
* Table.
|
|
*/
|
|
struct table_type {
|
|
TAILQ_ENTRY(table_type) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
enum rte_swx_table_match_type match_type;
|
|
struct rte_swx_table_ops ops;
|
|
};
|
|
|
|
TAILQ_HEAD(table_type_tailq, table_type);
|
|
|
|
struct match_field {
|
|
enum rte_swx_table_match_type match_type;
|
|
struct field *field;
|
|
};
|
|
|
|
struct table {
|
|
TAILQ_ENTRY(table) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
char args[RTE_SWX_NAME_SIZE];
|
|
struct table_type *type; /* NULL when n_fields == 0. */
|
|
|
|
/* Match. */
|
|
struct match_field *fields;
|
|
uint32_t n_fields;
|
|
struct header *header; /* Only valid when n_fields > 0. */
|
|
|
|
/* Action. */
|
|
struct action **actions;
|
|
struct action *default_action;
|
|
uint8_t *default_action_data;
|
|
uint32_t n_actions;
|
|
int default_action_is_const;
|
|
uint32_t action_data_size_max;
|
|
|
|
uint32_t size;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(table_tailq, table);
|
|
|
|
struct table_runtime {
|
|
rte_swx_table_lookup_t func;
|
|
void *mailbox;
|
|
uint8_t **key;
|
|
};
|
|
|
|
struct table_statistics {
|
|
uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
|
|
uint64_t *n_pkts_action;
|
|
};
|
|
|
|
/*
|
|
* Selector.
|
|
*/
|
|
struct selector {
|
|
TAILQ_ENTRY(selector) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
|
|
struct field *group_id_field;
|
|
struct field **selector_fields;
|
|
uint32_t n_selector_fields;
|
|
struct header *selector_header;
|
|
struct field *member_id_field;
|
|
|
|
uint32_t n_groups_max;
|
|
uint32_t n_members_per_group_max;
|
|
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(selector_tailq, selector);
|
|
|
|
struct selector_runtime {
|
|
void *mailbox;
|
|
uint8_t **group_id_buffer;
|
|
uint8_t **selector_buffer;
|
|
uint8_t **member_id_buffer;
|
|
};
|
|
|
|
struct selector_statistics {
|
|
uint64_t n_pkts;
|
|
};
|
|
|
|
/*
|
|
* Learner table.
|
|
*/
|
|
struct learner {
|
|
TAILQ_ENTRY(learner) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
|
|
/* Match. */
|
|
struct field **fields;
|
|
uint32_t n_fields;
|
|
struct header *header;
|
|
|
|
/* Action. */
|
|
struct action **actions;
|
|
struct action *default_action;
|
|
uint8_t *default_action_data;
|
|
uint32_t n_actions;
|
|
int default_action_is_const;
|
|
uint32_t action_data_size_max;
|
|
|
|
uint32_t size;
|
|
uint32_t timeout;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(learner_tailq, learner);
|
|
|
|
struct learner_runtime {
|
|
void *mailbox;
|
|
uint8_t **key;
|
|
};
|
|
|
|
struct learner_statistics {
|
|
uint64_t n_pkts_hit[2]; /* 0 = Miss, 1 = Hit. */
|
|
uint64_t n_pkts_learn[2]; /* 0 = Learn OK, 1 = Learn error. */
|
|
uint64_t n_pkts_forget;
|
|
uint64_t *n_pkts_action;
|
|
};
|
|
|
|
/*
|
|
* Register array.
|
|
*/
|
|
struct regarray {
|
|
TAILQ_ENTRY(regarray) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
uint64_t init_val;
|
|
uint32_t size;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(regarray_tailq, regarray);
|
|
|
|
struct regarray_runtime {
|
|
uint64_t *regarray;
|
|
uint32_t size_mask;
|
|
};
|
|
|
|
/*
|
|
* Meter array.
|
|
*/
|
|
struct meter_profile {
|
|
TAILQ_ENTRY(meter_profile) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
struct rte_meter_trtcm_params params;
|
|
struct rte_meter_trtcm_profile profile;
|
|
uint32_t n_users;
|
|
};
|
|
|
|
TAILQ_HEAD(meter_profile_tailq, meter_profile);
|
|
|
|
struct metarray {
|
|
TAILQ_ENTRY(metarray) node;
|
|
char name[RTE_SWX_NAME_SIZE];
|
|
uint32_t size;
|
|
uint32_t id;
|
|
};
|
|
|
|
TAILQ_HEAD(metarray_tailq, metarray);
|
|
|
|
struct meter {
|
|
struct rte_meter_trtcm m;
|
|
struct meter_profile *profile;
|
|
enum rte_color color_mask;
|
|
uint8_t pad[20];
|
|
|
|
uint64_t n_pkts[RTE_COLORS];
|
|
uint64_t n_bytes[RTE_COLORS];
|
|
};
|
|
|
|
struct metarray_runtime {
|
|
struct meter *metarray;
|
|
uint32_t size_mask;
|
|
};
|
|
|
|
/*
|
|
* Pipeline.
|
|
*/
|
|
struct thread {
|
|
/* Packet. */
|
|
struct rte_swx_pkt pkt;
|
|
uint8_t *ptr;
|
|
|
|
/* Structures. */
|
|
uint8_t **structs;
|
|
|
|
/* Packet headers. */
|
|
struct header_runtime *headers; /* Extracted or generated headers. */
|
|
struct header_out_runtime *headers_out; /* Emitted headers. */
|
|
uint8_t *header_storage;
|
|
uint8_t *header_out_storage;
|
|
uint64_t valid_headers;
|
|
uint32_t n_headers_out;
|
|
|
|
/* Packet meta-data. */
|
|
uint8_t *metadata;
|
|
|
|
/* Tables. */
|
|
struct table_runtime *tables;
|
|
struct selector_runtime *selectors;
|
|
struct learner_runtime *learners;
|
|
struct rte_swx_table_state *table_state;
|
|
uint64_t action_id;
|
|
int hit; /* 0 = Miss, 1 = Hit. */
|
|
uint32_t learner_id;
|
|
uint64_t time;
|
|
|
|
/* Extern objects and functions. */
|
|
struct extern_obj_runtime *extern_objs;
|
|
struct extern_func_runtime *extern_funcs;
|
|
|
|
/* Instructions. */
|
|
struct instruction *ip;
|
|
struct instruction *ret;
|
|
};
|
|
|
|
#define MASK64_BIT_GET(mask, pos) ((mask) & (1LLU << (pos)))
|
|
#define MASK64_BIT_SET(mask, pos) ((mask) | (1LLU << (pos)))
|
|
#define MASK64_BIT_CLR(mask, pos) ((mask) & ~(1LLU << (pos)))
|
|
|
|
#define HEADER_VALID(thread, header_id) \
|
|
MASK64_BIT_GET((thread)->valid_headers, header_id)
|
|
|
|
static inline uint64_t
|
|
instr_operand_hbo(struct thread *t, const struct instr_operand *x)
|
|
{
|
|
uint8_t *x_struct = t->structs[x->struct_id];
|
|
uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
|
|
uint64_t x64 = *x64_ptr;
|
|
uint64_t x64_mask = UINT64_MAX >> (64 - x->n_bits);
|
|
|
|
return x64 & x64_mask;
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline uint64_t
|
|
instr_operand_nbo(struct thread *t, const struct instr_operand *x)
|
|
{
|
|
uint8_t *x_struct = t->structs[x->struct_id];
|
|
uint64_t *x64_ptr = (uint64_t *)&x_struct[x->offset];
|
|
uint64_t x64 = *x64_ptr;
|
|
|
|
return ntoh64(x64) >> (64 - x->n_bits);
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_operand_nbo instr_operand_hbo
|
|
|
|
#endif
|
|
|
|
#define ALU(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = dst64 & dst64_mask; \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
|
|
uint64_t src = src64 & src64_mask; \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
#define ALU_MH(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = dst64 & dst64_mask; \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
|
|
}
|
|
|
|
#define ALU_HM(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
|
|
uint64_t src = src64 & src64_mask; \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | result; \
|
|
}
|
|
|
|
#define ALU_HM_FAST(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = dst64 & dst64_mask; \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->alu.src.n_bits); \
|
|
uint64_t src = hton64(src64 & src64_mask) >> (64 - (ip)->alu.dst.n_bits); \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | result; \
|
|
}
|
|
|
|
#define ALU_HH(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src = ntoh64(src64) >> (64 - (ip)->alu.src.n_bits); \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | result; \
|
|
}
|
|
|
|
#define ALU_HH_FAST(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = dst64 & dst64_mask; \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->alu.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->alu.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src = (src64 << (64 - (ip)->alu.src.n_bits)) >> (64 - (ip)->alu.dst.n_bits); \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | result; \
|
|
}
|
|
|
|
#else
|
|
|
|
#define ALU_MH ALU
|
|
#define ALU_HM ALU
|
|
#define ALU_HM_FAST ALU
|
|
#define ALU_HH ALU
|
|
#define ALU_HH_FAST ALU
|
|
|
|
#endif
|
|
|
|
#define ALU_I(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = dst64 & dst64_mask; \
|
|
\
|
|
uint64_t src = (ip)->alu.src_val; \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (result & dst64_mask); \
|
|
}
|
|
|
|
#define ALU_MI ALU_I
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
#define ALU_HI(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->alu.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->alu.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->alu.dst.n_bits); \
|
|
uint64_t dst = ntoh64(dst64) >> (64 - (ip)->alu.dst.n_bits); \
|
|
\
|
|
uint64_t src = (ip)->alu.src_val; \
|
|
\
|
|
uint64_t result = dst operator src; \
|
|
result = hton64(result << (64 - (ip)->alu.dst.n_bits)); \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | result; \
|
|
}
|
|
|
|
#else
|
|
|
|
#define ALU_HI ALU_I
|
|
|
|
#endif
|
|
|
|
#define MOV(thread, ip) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
|
|
uint64_t src = src64 & src64_mask; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
#define MOV_MH(thread, ip) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src = ntoh64(src64) >> (64 - (ip)->mov.src.n_bits); \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
|
|
}
|
|
|
|
#define MOV_HM(thread, ip) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->mov.src.n_bits); \
|
|
uint64_t src = src64 & src64_mask; \
|
|
\
|
|
src = hton64(src) >> (64 - (ip)->mov.dst.n_bits); \
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | src; \
|
|
}
|
|
|
|
#define MOV_HH(thread, ip) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
|
|
\
|
|
uint8_t *src_struct = (thread)->structs[(ip)->mov.src.struct_id]; \
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[(ip)->mov.src.offset]; \
|
|
uint64_t src64 = *src64_ptr; \
|
|
\
|
|
uint64_t src = src64 << (64 - (ip)->mov.src.n_bits); \
|
|
src = src >> (64 - (ip)->mov.dst.n_bits); \
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | src; \
|
|
}
|
|
|
|
#else
|
|
|
|
#define MOV_MH MOV
|
|
#define MOV_HM MOV
|
|
#define MOV_HH MOV
|
|
|
|
#endif
|
|
|
|
#define MOV_I(thread, ip) \
|
|
{ \
|
|
uint8_t *dst_struct = (thread)->structs[(ip)->mov.dst.struct_id]; \
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[(ip)->mov.dst.offset]; \
|
|
uint64_t dst64 = *dst64_ptr; \
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - (ip)->mov.dst.n_bits); \
|
|
\
|
|
uint64_t src = (ip)->mov.src_val; \
|
|
\
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask); \
|
|
}
|
|
|
|
#define JMP_CMP(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
|
|
uint64_t a = a64 & a64_mask; \
|
|
\
|
|
uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
|
|
uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
|
|
uint64_t b64 = *b64_ptr; \
|
|
uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
|
|
uint64_t b = b64 & b64_mask; \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
#define JMP_CMP_MH(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
|
|
uint64_t a = a64 & a64_mask; \
|
|
\
|
|
uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
|
|
uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
|
|
uint64_t b64 = *b64_ptr; \
|
|
uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#define JMP_CMP_HM(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
|
|
\
|
|
uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
|
|
uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
|
|
uint64_t b64 = *b64_ptr; \
|
|
uint64_t b64_mask = UINT64_MAX >> (64 - (ip)->jmp.b.n_bits); \
|
|
uint64_t b = b64 & b64_mask; \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#define JMP_CMP_HH(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
|
|
\
|
|
uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
|
|
uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
|
|
uint64_t b64 = *b64_ptr; \
|
|
uint64_t b = ntoh64(b64) >> (64 - (ip)->jmp.b.n_bits); \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#define JMP_CMP_HH_FAST(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a = a64 << (64 - (ip)->jmp.a.n_bits); \
|
|
\
|
|
uint8_t *b_struct = (thread)->structs[(ip)->jmp.b.struct_id]; \
|
|
uint64_t *b64_ptr = (uint64_t *)&b_struct[(ip)->jmp.b.offset]; \
|
|
uint64_t b64 = *b64_ptr; \
|
|
uint64_t b = b64 << (64 - (ip)->jmp.b.n_bits); \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#else
|
|
|
|
#define JMP_CMP_MH JMP_CMP
|
|
#define JMP_CMP_HM JMP_CMP
|
|
#define JMP_CMP_HH JMP_CMP
|
|
#define JMP_CMP_HH_FAST JMP_CMP
|
|
|
|
#endif
|
|
|
|
#define JMP_CMP_I(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a64_mask = UINT64_MAX >> (64 - (ip)->jmp.a.n_bits); \
|
|
uint64_t a = a64 & a64_mask; \
|
|
\
|
|
uint64_t b = (ip)->jmp.b_val; \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#define JMP_CMP_MI JMP_CMP_I
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
#define JMP_CMP_HI(thread, ip, operator) \
|
|
{ \
|
|
uint8_t *a_struct = (thread)->structs[(ip)->jmp.a.struct_id]; \
|
|
uint64_t *a64_ptr = (uint64_t *)&a_struct[(ip)->jmp.a.offset]; \
|
|
uint64_t a64 = *a64_ptr; \
|
|
uint64_t a = ntoh64(a64) >> (64 - (ip)->jmp.a.n_bits); \
|
|
\
|
|
uint64_t b = (ip)->jmp.b_val; \
|
|
\
|
|
(thread)->ip = (a operator b) ? (ip)->jmp.ip : ((thread)->ip + 1); \
|
|
}
|
|
|
|
#else
|
|
|
|
#define JMP_CMP_HI JMP_CMP_I
|
|
|
|
#endif
|
|
|
|
#define METADATA_READ(thread, offset, n_bits) \
|
|
({ \
|
|
uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
|
|
uint64_t m64 = *m64_ptr; \
|
|
uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
|
|
(m64 & m64_mask); \
|
|
})
|
|
|
|
#define METADATA_WRITE(thread, offset, n_bits, value) \
|
|
{ \
|
|
uint64_t *m64_ptr = (uint64_t *)&(thread)->metadata[offset]; \
|
|
uint64_t m64 = *m64_ptr; \
|
|
uint64_t m64_mask = UINT64_MAX >> (64 - (n_bits)); \
|
|
\
|
|
uint64_t m_new = value; \
|
|
\
|
|
*m64_ptr = (m64 & ~m64_mask) | (m_new & m64_mask); \
|
|
}
|
|
|
|
#ifndef RTE_SWX_PIPELINE_THREADS_MAX
|
|
#define RTE_SWX_PIPELINE_THREADS_MAX 16
|
|
#endif
|
|
|
|
#ifndef RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX
|
|
#define RTE_SWX_PIPELINE_INSTRUCTION_TABLE_SIZE_MAX 256
|
|
#endif
|
|
|
|
struct rte_swx_pipeline {
|
|
struct struct_type_tailq struct_types;
|
|
struct port_in_type_tailq port_in_types;
|
|
struct port_in_tailq ports_in;
|
|
struct port_out_type_tailq port_out_types;
|
|
struct port_out_tailq ports_out;
|
|
struct extern_type_tailq extern_types;
|
|
struct extern_obj_tailq extern_objs;
|
|
struct extern_func_tailq extern_funcs;
|
|
struct header_tailq headers;
|
|
struct struct_type *metadata_st;
|
|
uint32_t metadata_struct_id;
|
|
struct action_tailq actions;
|
|
struct table_type_tailq table_types;
|
|
struct table_tailq tables;
|
|
struct selector_tailq selectors;
|
|
struct learner_tailq learners;
|
|
struct regarray_tailq regarrays;
|
|
struct meter_profile_tailq meter_profiles;
|
|
struct metarray_tailq metarrays;
|
|
|
|
struct port_in_runtime *in;
|
|
struct port_out_runtime *out;
|
|
struct instruction **action_instructions;
|
|
action_func_t *action_funcs;
|
|
struct rte_swx_table_state *table_state;
|
|
struct table_statistics *table_stats;
|
|
struct selector_statistics *selector_stats;
|
|
struct learner_statistics *learner_stats;
|
|
struct regarray_runtime *regarray_runtime;
|
|
struct metarray_runtime *metarray_runtime;
|
|
struct instruction *instructions;
|
|
struct instruction_data *instruction_data;
|
|
instr_exec_t *instruction_table;
|
|
struct thread threads[RTE_SWX_PIPELINE_THREADS_MAX];
|
|
void *lib;
|
|
|
|
uint32_t n_structs;
|
|
uint32_t n_ports_in;
|
|
uint32_t n_ports_out;
|
|
uint32_t n_extern_objs;
|
|
uint32_t n_extern_funcs;
|
|
uint32_t n_actions;
|
|
uint32_t n_tables;
|
|
uint32_t n_selectors;
|
|
uint32_t n_learners;
|
|
uint32_t n_regarrays;
|
|
uint32_t n_metarrays;
|
|
uint32_t n_headers;
|
|
uint32_t thread_id;
|
|
uint32_t port_id;
|
|
uint32_t n_instructions;
|
|
int build_done;
|
|
int numa_node;
|
|
};
|
|
|
|
/*
|
|
* Instruction.
|
|
*/
|
|
static inline void
|
|
pipeline_port_inc(struct rte_swx_pipeline *p)
|
|
{
|
|
p->port_id = (p->port_id + 1) & (p->n_ports_in - 1);
|
|
}
|
|
|
|
static inline void
|
|
thread_ip_reset(struct rte_swx_pipeline *p, struct thread *t)
|
|
{
|
|
t->ip = p->instructions;
|
|
}
|
|
|
|
static inline void
|
|
thread_ip_set(struct thread *t, struct instruction *ip)
|
|
{
|
|
t->ip = ip;
|
|
}
|
|
|
|
static inline void
|
|
thread_ip_action_call(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
uint32_t action_id)
|
|
{
|
|
t->ret = t->ip + 1;
|
|
t->ip = p->action_instructions[action_id];
|
|
}
|
|
|
|
static inline void
|
|
thread_ip_inc(struct rte_swx_pipeline *p);
|
|
|
|
static inline void
|
|
thread_ip_inc(struct rte_swx_pipeline *p)
|
|
{
|
|
struct thread *t = &p->threads[p->thread_id];
|
|
|
|
t->ip++;
|
|
}
|
|
|
|
static inline void
|
|
thread_ip_inc_cond(struct thread *t, int cond)
|
|
{
|
|
t->ip += cond;
|
|
}
|
|
|
|
static inline void
|
|
thread_yield(struct rte_swx_pipeline *p)
|
|
{
|
|
p->thread_id = (p->thread_id + 1) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
|
|
}
|
|
|
|
static inline void
|
|
thread_yield_cond(struct rte_swx_pipeline *p, int cond)
|
|
{
|
|
p->thread_id = (p->thread_id + cond) & (RTE_SWX_PIPELINE_THREADS_MAX - 1);
|
|
}
|
|
|
|
/*
|
|
* rx.
|
|
*/
|
|
static inline int
|
|
__instr_rx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct port_in_runtime *port = &p->in[p->port_id];
|
|
struct rte_swx_pkt *pkt = &t->pkt;
|
|
int pkt_received;
|
|
|
|
/* Packet. */
|
|
pkt_received = port->pkt_rx(port->obj, pkt);
|
|
t->ptr = &pkt->pkt[pkt->offset];
|
|
rte_prefetch0(t->ptr);
|
|
|
|
TRACE("[Thread %2u] rx %s from port %u\n",
|
|
p->thread_id,
|
|
pkt_received ? "1 pkt" : "0 pkts",
|
|
p->port_id);
|
|
|
|
/* Headers. */
|
|
t->valid_headers = 0;
|
|
t->n_headers_out = 0;
|
|
|
|
/* Meta-data. */
|
|
METADATA_WRITE(t, ip->io.io.offset, ip->io.io.n_bits, p->port_id);
|
|
|
|
/* Tables. */
|
|
t->table_state = p->table_state;
|
|
|
|
/* Thread. */
|
|
pipeline_port_inc(p);
|
|
|
|
return pkt_received;
|
|
}
|
|
|
|
static inline void
|
|
instr_rx_exec(struct rte_swx_pipeline *p)
|
|
{
|
|
struct thread *t = &p->threads[p->thread_id];
|
|
struct instruction *ip = t->ip;
|
|
int pkt_received;
|
|
|
|
/* Packet. */
|
|
pkt_received = __instr_rx_exec(p, t, ip);
|
|
|
|
/* Thread. */
|
|
thread_ip_inc_cond(t, pkt_received);
|
|
thread_yield(p);
|
|
}
|
|
|
|
/*
|
|
* tx.
|
|
*/
|
|
static inline void
|
|
emit_handler(struct thread *t)
|
|
{
|
|
struct header_out_runtime *h0 = &t->headers_out[0];
|
|
struct header_out_runtime *h1 = &t->headers_out[1];
|
|
uint32_t offset = 0, i;
|
|
|
|
/* No header change or header decapsulation. */
|
|
if ((t->n_headers_out == 1) &&
|
|
(h0->ptr + h0->n_bytes == t->ptr)) {
|
|
TRACE("Emit handler: no header change or header decap.\n");
|
|
|
|
t->pkt.offset -= h0->n_bytes;
|
|
t->pkt.length += h0->n_bytes;
|
|
|
|
return;
|
|
}
|
|
|
|
/* Header encapsulation (optionally, with prior header decasulation). */
|
|
if ((t->n_headers_out == 2) &&
|
|
(h1->ptr + h1->n_bytes == t->ptr) &&
|
|
(h0->ptr == h0->ptr0)) {
|
|
uint32_t offset;
|
|
|
|
TRACE("Emit handler: header encapsulation.\n");
|
|
|
|
offset = h0->n_bytes + h1->n_bytes;
|
|
memcpy(t->ptr - offset, h0->ptr, h0->n_bytes);
|
|
t->pkt.offset -= offset;
|
|
t->pkt.length += offset;
|
|
|
|
return;
|
|
}
|
|
|
|
/* For any other case. */
|
|
TRACE("Emit handler: complex case.\n");
|
|
|
|
for (i = 0; i < t->n_headers_out; i++) {
|
|
struct header_out_runtime *h = &t->headers_out[i];
|
|
|
|
memcpy(&t->header_out_storage[offset], h->ptr, h->n_bytes);
|
|
offset += h->n_bytes;
|
|
}
|
|
|
|
if (offset) {
|
|
memcpy(t->ptr - offset, t->header_out_storage, offset);
|
|
t->pkt.offset -= offset;
|
|
t->pkt.length += offset;
|
|
}
|
|
}
|
|
|
|
static inline void
|
|
__instr_tx_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t port_id = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
|
|
struct port_out_runtime *port = &p->out[port_id];
|
|
struct rte_swx_pkt *pkt = &t->pkt;
|
|
|
|
TRACE("[Thread %2u]: tx 1 pkt to port %u\n",
|
|
p->thread_id,
|
|
(uint32_t)port_id);
|
|
|
|
/* Headers. */
|
|
emit_handler(t);
|
|
|
|
/* Packet. */
|
|
port->pkt_tx(port->obj, pkt);
|
|
}
|
|
|
|
static inline void
|
|
__instr_tx_i_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t port_id = ip->io.io.val;
|
|
struct port_out_runtime *port = &p->out[port_id];
|
|
struct rte_swx_pkt *pkt = &t->pkt;
|
|
|
|
TRACE("[Thread %2u]: tx (i) 1 pkt to port %u\n",
|
|
p->thread_id,
|
|
(uint32_t)port_id);
|
|
|
|
/* Headers. */
|
|
emit_handler(t);
|
|
|
|
/* Packet. */
|
|
port->pkt_tx(port->obj, pkt);
|
|
}
|
|
|
|
/*
|
|
* extract.
|
|
*/
|
|
static inline void
|
|
__instr_hdr_extract_many_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip,
|
|
uint32_t n_extract)
|
|
{
|
|
uint64_t valid_headers = t->valid_headers;
|
|
uint8_t *ptr = t->ptr;
|
|
uint32_t offset = t->pkt.offset;
|
|
uint32_t length = t->pkt.length;
|
|
uint32_t i;
|
|
|
|
for (i = 0; i < n_extract; i++) {
|
|
uint32_t header_id = ip->io.hdr.header_id[i];
|
|
uint32_t struct_id = ip->io.hdr.struct_id[i];
|
|
uint32_t n_bytes = ip->io.hdr.n_bytes[i];
|
|
|
|
TRACE("[Thread %2u]: extract header %u (%u bytes)\n",
|
|
p->thread_id,
|
|
header_id,
|
|
n_bytes);
|
|
|
|
/* Headers. */
|
|
t->structs[struct_id] = ptr;
|
|
valid_headers = MASK64_BIT_SET(valid_headers, header_id);
|
|
|
|
/* Packet. */
|
|
offset += n_bytes;
|
|
length -= n_bytes;
|
|
ptr += n_bytes;
|
|
}
|
|
|
|
/* Headers. */
|
|
t->valid_headers = valid_headers;
|
|
|
|
/* Packet. */
|
|
t->pkt.offset = offset;
|
|
t->pkt.length = length;
|
|
t->ptr = ptr;
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
__instr_hdr_extract_many_exec(p, t, ip, 1);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract2_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 2);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract3_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 3);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract4_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 4);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract5_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 5);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract6_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 6);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract7_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 7);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract8_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_extract_many_exec(p, t, ip, 8);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_extract_m_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t valid_headers = t->valid_headers;
|
|
uint8_t *ptr = t->ptr;
|
|
uint32_t offset = t->pkt.offset;
|
|
uint32_t length = t->pkt.length;
|
|
|
|
uint32_t n_bytes_last = METADATA_READ(t, ip->io.io.offset, ip->io.io.n_bits);
|
|
uint32_t header_id = ip->io.hdr.header_id[0];
|
|
uint32_t struct_id = ip->io.hdr.struct_id[0];
|
|
uint32_t n_bytes = ip->io.hdr.n_bytes[0];
|
|
|
|
struct header_runtime *h = &t->headers[header_id];
|
|
|
|
TRACE("[Thread %2u]: extract header %u (%u + %u bytes)\n",
|
|
p->thread_id,
|
|
header_id,
|
|
n_bytes,
|
|
n_bytes_last);
|
|
|
|
n_bytes += n_bytes_last;
|
|
|
|
/* Headers. */
|
|
t->structs[struct_id] = ptr;
|
|
t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
|
|
h->n_bytes = n_bytes;
|
|
|
|
/* Packet. */
|
|
t->pkt.offset = offset + n_bytes;
|
|
t->pkt.length = length - n_bytes;
|
|
t->ptr = ptr + n_bytes;
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_lookahead_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t valid_headers = t->valid_headers;
|
|
uint8_t *ptr = t->ptr;
|
|
|
|
uint32_t header_id = ip->io.hdr.header_id[0];
|
|
uint32_t struct_id = ip->io.hdr.struct_id[0];
|
|
|
|
TRACE("[Thread %2u]: lookahead header %u\n",
|
|
p->thread_id,
|
|
header_id);
|
|
|
|
/* Headers. */
|
|
t->structs[struct_id] = ptr;
|
|
t->valid_headers = MASK64_BIT_SET(valid_headers, header_id);
|
|
}
|
|
|
|
/*
|
|
* emit.
|
|
*/
|
|
static inline void
|
|
__instr_hdr_emit_many_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip,
|
|
uint32_t n_emit)
|
|
{
|
|
uint64_t valid_headers = t->valid_headers;
|
|
uint32_t n_headers_out = t->n_headers_out;
|
|
struct header_out_runtime *ho = &t->headers_out[n_headers_out - 1];
|
|
uint8_t *ho_ptr = NULL;
|
|
uint32_t ho_nbytes = 0, first = 1, i;
|
|
|
|
for (i = 0; i < n_emit; i++) {
|
|
uint32_t header_id = ip->io.hdr.header_id[i];
|
|
uint32_t struct_id = ip->io.hdr.struct_id[i];
|
|
|
|
struct header_runtime *hi = &t->headers[header_id];
|
|
uint8_t *hi_ptr0 = hi->ptr0;
|
|
uint32_t n_bytes = hi->n_bytes;
|
|
|
|
uint8_t *hi_ptr = t->structs[struct_id];
|
|
|
|
if (!MASK64_BIT_GET(valid_headers, header_id))
|
|
continue;
|
|
|
|
TRACE("[Thread %2u]: emit header %u\n",
|
|
p->thread_id,
|
|
header_id);
|
|
|
|
/* Headers. */
|
|
if (first) {
|
|
first = 0;
|
|
|
|
if (!t->n_headers_out) {
|
|
ho = &t->headers_out[0];
|
|
|
|
ho->ptr0 = hi_ptr0;
|
|
ho->ptr = hi_ptr;
|
|
|
|
ho_ptr = hi_ptr;
|
|
ho_nbytes = n_bytes;
|
|
|
|
n_headers_out = 1;
|
|
|
|
continue;
|
|
} else {
|
|
ho_ptr = ho->ptr;
|
|
ho_nbytes = ho->n_bytes;
|
|
}
|
|
}
|
|
|
|
if (ho_ptr + ho_nbytes == hi_ptr) {
|
|
ho_nbytes += n_bytes;
|
|
} else {
|
|
ho->n_bytes = ho_nbytes;
|
|
|
|
ho++;
|
|
ho->ptr0 = hi_ptr0;
|
|
ho->ptr = hi_ptr;
|
|
|
|
ho_ptr = hi_ptr;
|
|
ho_nbytes = n_bytes;
|
|
|
|
n_headers_out++;
|
|
}
|
|
}
|
|
|
|
ho->n_bytes = ho_nbytes;
|
|
t->n_headers_out = n_headers_out;
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
__instr_hdr_emit_many_exec(p, t, ip, 1);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 1);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit2_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 2);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit3_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 3);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit4_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 4);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit5_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 5);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit6_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 6);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit7_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 7);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_hdr_emit8_tx_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 9 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_hdr_emit_many_exec(p, t, ip, 8);
|
|
__instr_tx_exec(p, t, ip);
|
|
}
|
|
|
|
/*
|
|
* validate.
|
|
*/
|
|
static inline void
|
|
__instr_hdr_validate_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint32_t header_id = ip->valid.header_id;
|
|
|
|
TRACE("[Thread %2u] validate header %u\n", p->thread_id, header_id);
|
|
|
|
/* Headers. */
|
|
t->valid_headers = MASK64_BIT_SET(t->valid_headers, header_id);
|
|
}
|
|
|
|
/*
|
|
* invalidate.
|
|
*/
|
|
static inline void
|
|
__instr_hdr_invalidate_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint32_t header_id = ip->valid.header_id;
|
|
|
|
TRACE("[Thread %2u] invalidate header %u\n", p->thread_id, header_id);
|
|
|
|
/* Headers. */
|
|
t->valid_headers = MASK64_BIT_CLR(t->valid_headers, header_id);
|
|
}
|
|
|
|
/*
|
|
* learn.
|
|
*/
|
|
static inline void
|
|
__instr_learn_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t action_id = ip->learn.action_id;
|
|
uint32_t mf_offset = ip->learn.mf_offset;
|
|
uint32_t learner_id = t->learner_id;
|
|
struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
|
|
p->n_selectors + learner_id];
|
|
struct learner_runtime *l = &t->learners[learner_id];
|
|
struct learner_statistics *stats = &p->learner_stats[learner_id];
|
|
uint32_t status;
|
|
|
|
/* Table. */
|
|
status = rte_swx_table_learner_add(ts->obj,
|
|
l->mailbox,
|
|
t->time,
|
|
action_id,
|
|
&t->metadata[mf_offset]);
|
|
|
|
TRACE("[Thread %2u] learner %u learn %s\n",
|
|
p->thread_id,
|
|
learner_id,
|
|
status ? "ok" : "error");
|
|
|
|
stats->n_pkts_learn[status] += 1;
|
|
}
|
|
|
|
/*
|
|
* forget.
|
|
*/
|
|
static inline void
|
|
__instr_forget_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip __rte_unused)
|
|
{
|
|
uint32_t learner_id = t->learner_id;
|
|
struct rte_swx_table_state *ts = &t->table_state[p->n_tables +
|
|
p->n_selectors + learner_id];
|
|
struct learner_runtime *l = &t->learners[learner_id];
|
|
struct learner_statistics *stats = &p->learner_stats[learner_id];
|
|
|
|
/* Table. */
|
|
rte_swx_table_learner_delete(ts->obj, l->mailbox);
|
|
|
|
TRACE("[Thread %2u] learner %u forget\n",
|
|
p->thread_id,
|
|
learner_id);
|
|
|
|
stats->n_pkts_forget += 1;
|
|
}
|
|
|
|
/*
|
|
* extern.
|
|
*/
|
|
static inline uint32_t
|
|
__instr_extern_obj_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint32_t obj_id = ip->ext_obj.ext_obj_id;
|
|
uint32_t func_id = ip->ext_obj.func_id;
|
|
struct extern_obj_runtime *obj = &t->extern_objs[obj_id];
|
|
rte_swx_extern_type_member_func_t func = obj->funcs[func_id];
|
|
uint32_t done;
|
|
|
|
TRACE("[Thread %2u] extern obj %u member func %u\n",
|
|
p->thread_id,
|
|
obj_id,
|
|
func_id);
|
|
|
|
done = func(obj->obj, obj->mailbox);
|
|
|
|
return done;
|
|
}
|
|
|
|
static inline uint32_t
|
|
__instr_extern_func_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint32_t ext_func_id = ip->ext_func.ext_func_id;
|
|
struct extern_func_runtime *ext_func = &t->extern_funcs[ext_func_id];
|
|
rte_swx_extern_func_t func = ext_func->func;
|
|
uint32_t done;
|
|
|
|
TRACE("[Thread %2u] extern func %u\n",
|
|
p->thread_id,
|
|
ext_func_id);
|
|
|
|
done = func(ext_func->mailbox);
|
|
|
|
return done;
|
|
}
|
|
|
|
/*
|
|
* mov.
|
|
*/
|
|
static inline void
|
|
__instr_mov_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] mov\n", p->thread_id);
|
|
|
|
MOV(t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_mov_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] mov (mh)\n", p->thread_id);
|
|
|
|
MOV_MH(t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_mov_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] mov (hm)\n", p->thread_id);
|
|
|
|
MOV_HM(t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_mov_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] mov (hh)\n", p->thread_id);
|
|
|
|
MOV_HH(t, ip);
|
|
}
|
|
|
|
static inline void
|
|
__instr_mov_i_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] mov m.f %" PRIx64 "\n", p->thread_id, ip->mov.src_val);
|
|
|
|
MOV_I(t, ip);
|
|
}
|
|
|
|
/*
|
|
* dma.
|
|
*/
|
|
static inline void
|
|
__instr_dma_ht_many_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip,
|
|
uint32_t n_dma)
|
|
{
|
|
uint8_t *action_data = t->structs[0];
|
|
uint64_t valid_headers = t->valid_headers;
|
|
uint32_t i;
|
|
|
|
for (i = 0; i < n_dma; i++) {
|
|
uint32_t header_id = ip->dma.dst.header_id[i];
|
|
uint32_t struct_id = ip->dma.dst.struct_id[i];
|
|
uint32_t offset = ip->dma.src.offset[i];
|
|
uint32_t n_bytes = ip->dma.n_bytes[i];
|
|
|
|
struct header_runtime *h = &t->headers[header_id];
|
|
uint8_t *h_ptr0 = h->ptr0;
|
|
uint8_t *h_ptr = t->structs[struct_id];
|
|
|
|
void *dst = MASK64_BIT_GET(valid_headers, header_id) ?
|
|
h_ptr : h_ptr0;
|
|
void *src = &action_data[offset];
|
|
|
|
TRACE("[Thread %2u] dma h.s t.f\n", p->thread_id);
|
|
|
|
/* Headers. */
|
|
memcpy(dst, src, n_bytes);
|
|
t->structs[struct_id] = dst;
|
|
valid_headers = MASK64_BIT_SET(valid_headers, header_id);
|
|
}
|
|
|
|
t->valid_headers = valid_headers;
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
__instr_dma_ht_many_exec(p, t, ip, 1);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht2_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 2 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 2);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht3_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 3 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 3);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht4_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 4 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 4);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht5_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 5 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 5);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht6_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 6 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 6);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht7_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 7 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 7);
|
|
}
|
|
|
|
static inline void
|
|
__instr_dma_ht8_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] *** The next 8 instructions are fused. ***\n", p->thread_id);
|
|
|
|
__instr_dma_ht_many_exec(p, t, ip, 8);
|
|
}
|
|
|
|
/*
|
|
* alu.
|
|
*/
|
|
static inline void
|
|
__instr_alu_add_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add\n", p->thread_id);
|
|
|
|
ALU(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_add_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_add_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add (hm)\n", p->thread_id);
|
|
|
|
ALU_HM(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_add_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add (hh)\n", p->thread_id);
|
|
|
|
ALU_HH(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_add_mi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add (mi)\n", p->thread_id);
|
|
|
|
ALU_MI(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_add_hi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] add (hi)\n", p->thread_id);
|
|
|
|
ALU_HI(t, ip, +);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub\n", p->thread_id);
|
|
|
|
ALU(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub (hm)\n", p->thread_id);
|
|
|
|
ALU_HM(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub (hh)\n", p->thread_id);
|
|
|
|
ALU_HH(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_mi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub (mi)\n", p->thread_id);
|
|
|
|
ALU_MI(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_sub_hi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] sub (hi)\n", p->thread_id);
|
|
|
|
ALU_HI(t, ip, -);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl\n", p->thread_id);
|
|
|
|
ALU(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl (hm)\n", p->thread_id);
|
|
|
|
ALU_HM(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl (hh)\n", p->thread_id);
|
|
|
|
ALU_HH(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_mi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl (mi)\n", p->thread_id);
|
|
|
|
ALU_MI(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shl_hi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shl (hi)\n", p->thread_id);
|
|
|
|
ALU_HI(t, ip, <<);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr\n", p->thread_id);
|
|
|
|
ALU(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr (hm)\n", p->thread_id);
|
|
|
|
ALU_HM(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr (hh)\n", p->thread_id);
|
|
|
|
ALU_HH(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_mi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr (mi)\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
ALU_MI(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_shr_hi_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] shr (hi)\n", p->thread_id);
|
|
|
|
ALU_HI(t, ip, >>);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_and_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] and\n", p->thread_id);
|
|
|
|
ALU(t, ip, &);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_and_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] and (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, &);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_and_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] and (hm)\n", p->thread_id);
|
|
|
|
ALU_HM_FAST(t, ip, &);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_and_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] and (hh)\n", p->thread_id);
|
|
|
|
ALU_HH_FAST(t, ip, &);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_and_i_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] and (i)\n", p->thread_id);
|
|
|
|
ALU_I(t, ip, &);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_or_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] or\n", p->thread_id);
|
|
|
|
ALU(t, ip, |);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_or_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] or (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, |);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_or_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] or (hm)\n", p->thread_id);
|
|
|
|
ALU_HM_FAST(t, ip, |);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_or_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] or (hh)\n", p->thread_id);
|
|
|
|
ALU_HH_FAST(t, ip, |);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_or_i_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] or (i)\n", p->thread_id);
|
|
|
|
ALU_I(t, ip, |);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_xor_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] xor\n", p->thread_id);
|
|
|
|
ALU(t, ip, ^);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_xor_mh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] xor (mh)\n", p->thread_id);
|
|
|
|
ALU_MH(t, ip, ^);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_xor_hm_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] xor (hm)\n", p->thread_id);
|
|
|
|
ALU_HM_FAST(t, ip, ^);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_xor_hh_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] xor (hh)\n", p->thread_id);
|
|
|
|
ALU_HH_FAST(t, ip, ^);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_xor_i_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
TRACE("[Thread %2u] xor (i)\n", p->thread_id);
|
|
|
|
ALU_I(t, ip, ^);
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_ckadd_field_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint8_t *dst_struct, *src_struct;
|
|
uint16_t *dst16_ptr, dst;
|
|
uint64_t *src64_ptr, src64, src64_mask, src;
|
|
uint64_t r;
|
|
|
|
TRACE("[Thread %2u] ckadd (field)\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
dst_struct = t->structs[ip->alu.dst.struct_id];
|
|
dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
|
|
dst = *dst16_ptr;
|
|
|
|
src_struct = t->structs[ip->alu.src.struct_id];
|
|
src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
|
|
src64 = *src64_ptr;
|
|
src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
|
|
src = src64 & src64_mask;
|
|
|
|
r = dst;
|
|
r = ~r & 0xFFFF;
|
|
|
|
/* The first input (r) is a 16-bit number. The second and the third
|
|
* inputs are 32-bit numbers. In the worst case scenario, the sum of the
|
|
* three numbers (output r) is a 34-bit number.
|
|
*/
|
|
r += (src >> 32) + (src & 0xFFFFFFFF);
|
|
|
|
/* The first input is a 16-bit number. The second input is an 18-bit
|
|
* number. In the worst case scenario, the sum of the two numbers is a
|
|
* 19-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
|
|
* a 3-bit number (0 .. 7). Their sum is a 17-bit number (0 .. 0x10006).
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* When the input r is (0 .. 0xFFFF), the output r is equal to the input
|
|
* r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
|
|
* 0x10006), the output r is (0 .. 7). So no carry bit can be generated,
|
|
* therefore the output r is always a 16-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
r = ~r & 0xFFFF;
|
|
r = r ? r : 0xFFFF;
|
|
|
|
*dst16_ptr = (uint16_t)r;
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_cksub_field_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint8_t *dst_struct, *src_struct;
|
|
uint16_t *dst16_ptr, dst;
|
|
uint64_t *src64_ptr, src64, src64_mask, src;
|
|
uint64_t r;
|
|
|
|
TRACE("[Thread %2u] cksub (field)\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
dst_struct = t->structs[ip->alu.dst.struct_id];
|
|
dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
|
|
dst = *dst16_ptr;
|
|
|
|
src_struct = t->structs[ip->alu.src.struct_id];
|
|
src64_ptr = (uint64_t *)&src_struct[ip->alu.src.offset];
|
|
src64 = *src64_ptr;
|
|
src64_mask = UINT64_MAX >> (64 - ip->alu.src.n_bits);
|
|
src = src64 & src64_mask;
|
|
|
|
r = dst;
|
|
r = ~r & 0xFFFF;
|
|
|
|
/* Subtraction in 1's complement arithmetic (i.e. a '- b) is the same as
|
|
* the following sequence of operations in 2's complement arithmetic:
|
|
* a '- b = (a - b) % 0xFFFF.
|
|
*
|
|
* In order to prevent an underflow for the below subtraction, in which
|
|
* a 33-bit number (the subtrahend) is taken out of a 16-bit number (the
|
|
* minuend), we first add a multiple of the 0xFFFF modulus to the
|
|
* minuend. The number we add to the minuend needs to be a 34-bit number
|
|
* or higher, so for readability reasons we picked the 36-bit multiple.
|
|
* We are effectively turning the 16-bit minuend into a 36-bit number:
|
|
* (a - b) % 0xFFFF = (a + 0xFFFF00000 - b) % 0xFFFF.
|
|
*/
|
|
r += 0xFFFF00000ULL; /* The output r is a 36-bit number. */
|
|
|
|
/* A 33-bit number is subtracted from a 36-bit number (the input r). The
|
|
* result (the output r) is a 36-bit number.
|
|
*/
|
|
r -= (src >> 32) + (src & 0xFFFFFFFF);
|
|
|
|
/* The first input is a 16-bit number. The second input is a 20-bit
|
|
* number. Their sum is a 21-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
|
|
* a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1001E).
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* When the input r is (0 .. 0xFFFF), the output r is equal to the input
|
|
* r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
|
|
* 0x1001E), the output r is (0 .. 31). So no carry bit can be
|
|
* generated, therefore the output r is always a 16-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
r = ~r & 0xFFFF;
|
|
r = r ? r : 0xFFFF;
|
|
|
|
*dst16_ptr = (uint16_t)r;
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_ckadd_struct20_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint8_t *dst_struct, *src_struct;
|
|
uint16_t *dst16_ptr;
|
|
uint32_t *src32_ptr;
|
|
uint64_t r0, r1;
|
|
|
|
TRACE("[Thread %2u] ckadd (struct of 20 bytes)\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
dst_struct = t->structs[ip->alu.dst.struct_id];
|
|
dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
|
|
|
|
src_struct = t->structs[ip->alu.src.struct_id];
|
|
src32_ptr = (uint32_t *)&src_struct[0];
|
|
|
|
r0 = src32_ptr[0]; /* r0 is a 32-bit number. */
|
|
r1 = src32_ptr[1]; /* r1 is a 32-bit number. */
|
|
r0 += src32_ptr[2]; /* The output r0 is a 33-bit number. */
|
|
r1 += src32_ptr[3]; /* The output r1 is a 33-bit number. */
|
|
r0 += r1 + src32_ptr[4]; /* The output r0 is a 35-bit number. */
|
|
|
|
/* The first input is a 16-bit number. The second input is a 19-bit
|
|
* number. Their sum is a 20-bit number.
|
|
*/
|
|
r0 = (r0 & 0xFFFF) + (r0 >> 16);
|
|
|
|
/* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
|
|
* a 4-bit number (0 .. 15). The sum is a 17-bit number (0 .. 0x1000E).
|
|
*/
|
|
r0 = (r0 & 0xFFFF) + (r0 >> 16);
|
|
|
|
/* When the input r is (0 .. 0xFFFF), the output r is equal to the input
|
|
* r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
|
|
* 0x1000E), the output r is (0 .. 15). So no carry bit can be
|
|
* generated, therefore the output r is always a 16-bit number.
|
|
*/
|
|
r0 = (r0 & 0xFFFF) + (r0 >> 16);
|
|
|
|
r0 = ~r0 & 0xFFFF;
|
|
r0 = r0 ? r0 : 0xFFFF;
|
|
|
|
*dst16_ptr = (uint16_t)r0;
|
|
}
|
|
|
|
static inline void
|
|
__instr_alu_ckadd_struct_exec(struct rte_swx_pipeline *p __rte_unused,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint8_t *dst_struct, *src_struct;
|
|
uint16_t *dst16_ptr;
|
|
uint32_t *src32_ptr;
|
|
uint64_t r = 0;
|
|
uint32_t i;
|
|
|
|
TRACE("[Thread %2u] ckadd (struct)\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
dst_struct = t->structs[ip->alu.dst.struct_id];
|
|
dst16_ptr = (uint16_t *)&dst_struct[ip->alu.dst.offset];
|
|
|
|
src_struct = t->structs[ip->alu.src.struct_id];
|
|
src32_ptr = (uint32_t *)&src_struct[0];
|
|
|
|
/* The max number of 32-bit words in a 256-byte header is 8 = 2^3.
|
|
* Therefore, in the worst case scenario, a 35-bit number is added to a
|
|
* 16-bit number (the input r), so the output r is 36-bit number.
|
|
*/
|
|
for (i = 0; i < ip->alu.src.n_bits / 32; i++, src32_ptr++)
|
|
r += *src32_ptr;
|
|
|
|
/* The first input is a 16-bit number. The second input is a 20-bit
|
|
* number. Their sum is a 21-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* The first input is a 16-bit number (0 .. 0xFFFF). The second input is
|
|
* a 5-bit number (0 .. 31). The sum is a 17-bit number (0 .. 0x1000E).
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
/* When the input r is (0 .. 0xFFFF), the output r is equal to the input
|
|
* r, so the output is (0 .. 0xFFFF). When the input r is (0x10000 ..
|
|
* 0x1001E), the output r is (0 .. 31). So no carry bit can be
|
|
* generated, therefore the output r is always a 16-bit number.
|
|
*/
|
|
r = (r & 0xFFFF) + (r >> 16);
|
|
|
|
r = ~r & 0xFFFF;
|
|
r = r ? r : 0xFFFF;
|
|
|
|
*dst16_ptr = (uint16_t)r;
|
|
}
|
|
|
|
/*
|
|
* Register array.
|
|
*/
|
|
static inline uint64_t *
|
|
instr_regarray_regarray(struct rte_swx_pipeline *p, const struct instruction *ip)
|
|
{
|
|
struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
|
|
return r->regarray;
|
|
}
|
|
|
|
static inline uint64_t
|
|
instr_regarray_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
|
|
|
|
uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
|
|
uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
|
|
uint64_t idx64 = *idx64_ptr;
|
|
uint64_t idx64_mask = UINT64_MAX >> (64 - ip->regarray.idx.n_bits);
|
|
uint64_t idx = idx64 & idx64_mask & r->size_mask;
|
|
|
|
return idx;
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline uint64_t
|
|
instr_regarray_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
|
|
|
|
uint8_t *idx_struct = t->structs[ip->regarray.idx.struct_id];
|
|
uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->regarray.idx.offset];
|
|
uint64_t idx64 = *idx64_ptr;
|
|
uint64_t idx = (ntoh64(idx64) >> (64 - ip->regarray.idx.n_bits)) & r->size_mask;
|
|
|
|
return idx;
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_regarray_idx_nbo instr_regarray_idx_hbo
|
|
|
|
#endif
|
|
|
|
static inline uint64_t
|
|
instr_regarray_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
|
|
{
|
|
struct regarray_runtime *r = &p->regarray_runtime[ip->regarray.regarray_id];
|
|
|
|
uint64_t idx = ip->regarray.idx_val & r->size_mask;
|
|
|
|
return idx;
|
|
}
|
|
|
|
static inline uint64_t
|
|
instr_regarray_src_hbo(struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
|
|
uint64_t src64 = *src64_ptr;
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
|
|
uint64_t src = src64 & src64_mask;
|
|
|
|
return src;
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline uint64_t
|
|
instr_regarray_src_nbo(struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint8_t *src_struct = t->structs[ip->regarray.dstsrc.struct_id];
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->regarray.dstsrc.offset];
|
|
uint64_t src64 = *src64_ptr;
|
|
uint64_t src = ntoh64(src64) >> (64 - ip->regarray.dstsrc.n_bits);
|
|
|
|
return src;
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_regarray_src_nbo instr_regarray_src_hbo
|
|
|
|
#endif
|
|
|
|
static inline void
|
|
instr_regarray_dst_hbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
|
|
{
|
|
uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
|
|
uint64_t dst64 = *dst64_ptr;
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
|
|
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
|
|
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline void
|
|
instr_regarray_dst_nbo_src_hbo_set(struct thread *t, const struct instruction *ip, uint64_t src)
|
|
{
|
|
uint8_t *dst_struct = t->structs[ip->regarray.dstsrc.struct_id];
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->regarray.dstsrc.offset];
|
|
uint64_t dst64 = *dst64_ptr;
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - ip->regarray.dstsrc.n_bits);
|
|
|
|
src = hton64(src) >> (64 - ip->regarray.dstsrc.n_bits);
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_regarray_dst_nbo_src_hbo_set instr_regarray_dst_hbo_src_hbo_set
|
|
|
|
#endif
|
|
|
|
static inline void
|
|
__instr_regprefetch_rh_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regprefetch (r[h])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
rte_prefetch0(®array[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regprefetch_rm_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regprefetch (r[m])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
rte_prefetch0(®array[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regprefetch_ri_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t __rte_unused,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regprefetch (r[i])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
rte_prefetch0(®array[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_hrh_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (h = r[h])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_hrm_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (h = r[m])\n", p->thread_id);
|
|
|
|
/* Structs. */
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_mrh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (m = r[h])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_mrm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (m = r[m])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_hri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (h = r[i])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
instr_regarray_dst_nbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regrd_mri_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx;
|
|
|
|
TRACE("[Thread %2u] regrd (m = r[i])\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
instr_regarray_dst_hbo_src_hbo_set(t, ip, regarray[idx]);
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[h] = h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[h] = m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[m] = h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[m] = m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[h] = i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[m] = i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[i] = h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[i] = m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regwr_rii_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t __rte_unused,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regwr (r[i] = i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] = src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rhh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[h] += h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[h] += m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rmh_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[m] += h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[m] += m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[h] += i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_nbo(p, t, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[m] += i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_hbo(p, t, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rih_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[i] += h)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = instr_regarray_src_nbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rim_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[i] += m)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = instr_regarray_src_hbo(t, ip);
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
static inline void
|
|
__instr_regadd_rii_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t __rte_unused,
|
|
const struct instruction *ip)
|
|
{
|
|
uint64_t *regarray, idx, src;
|
|
|
|
TRACE("[Thread %2u] regadd (r[i] += i)\n", p->thread_id);
|
|
|
|
regarray = instr_regarray_regarray(p, ip);
|
|
idx = instr_regarray_idx_imm(p, ip);
|
|
src = ip->regarray.dstsrc_val;
|
|
regarray[idx] += src;
|
|
}
|
|
|
|
/*
|
|
* metarray.
|
|
*/
|
|
static inline struct meter *
|
|
instr_meter_idx_hbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
|
|
|
|
uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
|
|
uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
|
|
uint64_t idx64 = *idx64_ptr;
|
|
uint64_t idx64_mask = UINT64_MAX >> (64 - (ip)->meter.idx.n_bits);
|
|
uint64_t idx = idx64 & idx64_mask & r->size_mask;
|
|
|
|
return &r->metarray[idx];
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline struct meter *
|
|
instr_meter_idx_nbo(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
|
|
|
|
uint8_t *idx_struct = t->structs[ip->meter.idx.struct_id];
|
|
uint64_t *idx64_ptr = (uint64_t *)&idx_struct[ip->meter.idx.offset];
|
|
uint64_t idx64 = *idx64_ptr;
|
|
uint64_t idx = (ntoh64(idx64) >> (64 - ip->meter.idx.n_bits)) & r->size_mask;
|
|
|
|
return &r->metarray[idx];
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_meter_idx_nbo instr_meter_idx_hbo
|
|
|
|
#endif
|
|
|
|
static inline struct meter *
|
|
instr_meter_idx_imm(struct rte_swx_pipeline *p, const struct instruction *ip)
|
|
{
|
|
struct metarray_runtime *r = &p->metarray_runtime[ip->meter.metarray_id];
|
|
|
|
uint64_t idx = ip->meter.idx_val & r->size_mask;
|
|
|
|
return &r->metarray[idx];
|
|
}
|
|
|
|
static inline uint32_t
|
|
instr_meter_length_hbo(struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
|
|
uint64_t src64 = *src64_ptr;
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - (ip)->meter.length.n_bits);
|
|
uint64_t src = src64 & src64_mask;
|
|
|
|
return (uint32_t)src;
|
|
}
|
|
|
|
#if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
|
|
|
|
static inline uint32_t
|
|
instr_meter_length_nbo(struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint8_t *src_struct = t->structs[ip->meter.length.struct_id];
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.length.offset];
|
|
uint64_t src64 = *src64_ptr;
|
|
uint64_t src = ntoh64(src64) >> (64 - ip->meter.length.n_bits);
|
|
|
|
return (uint32_t)src;
|
|
}
|
|
|
|
#else
|
|
|
|
#define instr_meter_length_nbo instr_meter_length_hbo
|
|
|
|
#endif
|
|
|
|
static inline enum rte_color
|
|
instr_meter_color_in_hbo(struct thread *t, const struct instruction *ip)
|
|
{
|
|
uint8_t *src_struct = t->structs[ip->meter.color_in.struct_id];
|
|
uint64_t *src64_ptr = (uint64_t *)&src_struct[ip->meter.color_in.offset];
|
|
uint64_t src64 = *src64_ptr;
|
|
uint64_t src64_mask = UINT64_MAX >> (64 - ip->meter.color_in.n_bits);
|
|
uint64_t src = src64 & src64_mask;
|
|
|
|
return (enum rte_color)src;
|
|
}
|
|
|
|
static inline void
|
|
instr_meter_color_out_hbo_set(struct thread *t,
|
|
const struct instruction *ip,
|
|
enum rte_color color_out)
|
|
{
|
|
uint8_t *dst_struct = t->structs[ip->meter.color_out.struct_id];
|
|
uint64_t *dst64_ptr = (uint64_t *)&dst_struct[ip->meter.color_out.offset];
|
|
uint64_t dst64 = *dst64_ptr;
|
|
uint64_t dst64_mask = UINT64_MAX >> (64 - ip->meter.color_out.n_bits);
|
|
|
|
uint64_t src = (uint64_t)color_out;
|
|
|
|
*dst64_ptr = (dst64 & ~dst64_mask) | (src & dst64_mask);
|
|
}
|
|
|
|
static inline void
|
|
__instr_metprefetch_h_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
|
|
TRACE("[Thread %2u] metprefetch (h)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_nbo(p, t, ip);
|
|
rte_prefetch0(m);
|
|
}
|
|
|
|
static inline void
|
|
__instr_metprefetch_m_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t,
|
|
const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
|
|
TRACE("[Thread %2u] metprefetch (m)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_hbo(p, t, ip);
|
|
rte_prefetch0(m);
|
|
}
|
|
|
|
static inline void
|
|
__instr_metprefetch_i_exec(struct rte_swx_pipeline *p,
|
|
struct thread *t __rte_unused,
|
|
const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
|
|
TRACE("[Thread %2u] metprefetch (i)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_imm(p, ip);
|
|
rte_prefetch0(m);
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_hhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (hhm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_nbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_hhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (hhi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_nbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_hmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (hmm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_nbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_hmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (hmi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_nbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_mhm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (mhm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_hbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_mhi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (mhi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_hbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_mmm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (mmm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_hbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_mmi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (mmi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_hbo(p, t, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_ihm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (ihm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_imm(p, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_ihi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (ihi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_imm(p, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_nbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_imm_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (imm)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_imm(p, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = instr_meter_color_in_hbo(t, ip);
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
static inline void
|
|
__instr_meter_imi_exec(struct rte_swx_pipeline *p, struct thread *t, const struct instruction *ip)
|
|
{
|
|
struct meter *m;
|
|
uint64_t time, n_pkts, n_bytes;
|
|
uint32_t length;
|
|
enum rte_color color_in, color_out;
|
|
|
|
TRACE("[Thread %2u] meter (imi)\n", p->thread_id);
|
|
|
|
m = instr_meter_idx_imm(p, ip);
|
|
rte_prefetch0(m->n_pkts);
|
|
time = rte_get_tsc_cycles();
|
|
length = instr_meter_length_hbo(t, ip);
|
|
color_in = (enum rte_color)ip->meter.color_in_val;
|
|
|
|
color_out = rte_meter_trtcm_color_aware_check(&m->m,
|
|
&m->profile->profile,
|
|
time,
|
|
length,
|
|
color_in);
|
|
|
|
color_out &= m->color_mask;
|
|
|
|
n_pkts = m->n_pkts[color_out];
|
|
n_bytes = m->n_bytes[color_out];
|
|
|
|
instr_meter_color_out_hbo_set(t, ip, color_out);
|
|
|
|
m->n_pkts[color_out] = n_pkts + 1;
|
|
m->n_bytes[color_out] = n_bytes + length;
|
|
}
|
|
|
|
#endif
|