d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
b5886e64fc
numam-dpdk/lib/bpf/bpf_exec.c
Bruce Richardson 99a2dd955f lib: remove librte_ prefix from directory names 
There is no reason for the DPDK libraries to all have 'librte_' prefix on
the directory names. This prefix makes the directory names longer and also
makes it awkward to add features referring to individual libraries in the
build - should the lib names be specified with or without the prefix.
Therefore, we can just remove the library prefix and use the library's
unique name as the directory name, i.e. 'eal' rather than 'librte_eal'

Signed-off-by: Bruce Richardson <bruce.richardson@intel.com>
2021-04-21 14:04:09 +02:00

511 lines
14 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <stdint.h>
#include <inttypes.h>
#include <rte_common.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_memory.h>
#include <rte_eal.h>
#include <rte_byteorder.h>
#include "bpf_impl.h"
#define BPF_JMP_UNC(ins) ((ins) += (ins)->off)
#define BPF_JMP_CND_REG(reg, ins, op, type) \
((ins) += \
((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
(ins)->off : 0)
#define BPF_JMP_CND_IMM(reg, ins, op, type) \
((ins) += \
((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
(ins)->off : 0)
#define BPF_NEG_ALU(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
#define EBPF_MOV_ALU_REG(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
#define BPF_OP_ALU_REG(reg, ins, op, type) \
((reg)[(ins)->dst_reg] = \
(type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
#define EBPF_MOV_ALU_IMM(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(ins)->imm)
#define BPF_OP_ALU_IMM(reg, ins, op, type) \
((reg)[(ins)->dst_reg] = \
(type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
#define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \
if ((type)(reg)[(ins)->src_reg] == 0) { \
RTE_BPF_LOG(ERR, \
"%s(%p): division by 0 at pc: %#zx;\n", \
__func__, bpf, \
(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
return 0; \
} \
} while (0)
#define BPF_LD_REG(reg, ins, type) \
((reg)[(ins)->dst_reg] = \
*(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
#define BPF_ST_IMM(reg, ins, type) \
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(ins)->imm)
#define BPF_ST_REG(reg, ins, type) \
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(reg)[(ins)->src_reg])
#define BPF_ST_XADD_REG(reg, ins, tp) \
(rte_atomic##tp##_add((rte_atomic##tp##_t *) \
(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
reg[ins->src_reg]))
/* BPF_LD | BPF_ABS/BPF_IND */
#define NOP(x) (x)
#define BPF_LD_ABS(bpf, reg, ins, type, op) do { \
const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
if (p == NULL) \
return 0; \
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
#define BPF_LD_IND(bpf, reg, ins, type, op) do { \
uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
if (p == NULL) \
return 0; \
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
static inline void
bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
{
uint64_t *v;
v = reg + ins->dst_reg;
switch (ins->imm) {
case 16:
*v = rte_cpu_to_be_16(*v);
break;
case 32:
*v = rte_cpu_to_be_32(*v);
break;
case 64:
*v = rte_cpu_to_be_64(*v);
break;
}
}
static inline void
bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
{
uint64_t *v;
v = reg + ins->dst_reg;
switch (ins->imm) {
case 16:
*v = rte_cpu_to_le_16(*v);
break;
case 32:
*v = rte_cpu_to_le_32(*v);
break;
case 64:
*v = rte_cpu_to_le_64(*v);
break;
}
}
static inline const void *
bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
const struct ebpf_insn *ins, uint32_t off, uint32_t len)
{
const struct rte_mbuf *mb;
const void *p;
mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6];
p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0);
if (p == NULL)
RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
"load beyond packet boundary at pc: %#zx;\n",
__func__, bpf, mb, off, len,
(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
return p;
}
static inline uint64_t
bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
{
const struct ebpf_insn *ins;
for (ins = bpf->prm.ins; ; ins++) {
switch (ins->code) {
/* 32 bit ALU IMM operations */
case (BPF_ALU | BPF_ADD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, +, uint32_t);
break;
case (BPF_ALU | BPF_SUB | BPF_K):
BPF_OP_ALU_IMM(reg, ins, -, uint32_t);
break;
case (BPF_ALU | BPF_AND | BPF_K):
BPF_OP_ALU_IMM(reg, ins, &, uint32_t);
break;
case (BPF_ALU | BPF_OR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, |, uint32_t);
break;
case (BPF_ALU | BPF_LSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);
break;
case (BPF_ALU | BPF_RSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);
break;
case (BPF_ALU | BPF_XOR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
break;
case (BPF_ALU | BPF_MUL | BPF_K):
BPF_OP_ALU_IMM(reg, ins, *, uint32_t);
break;
case (BPF_ALU | BPF_DIV | BPF_K):
BPF_OP_ALU_IMM(reg, ins, /, uint32_t);
break;
case (BPF_ALU | BPF_MOD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, %, uint32_t);
break;
case (BPF_ALU | EBPF_MOV | BPF_K):
EBPF_MOV_ALU_IMM(reg, ins, uint32_t);
break;
/* 32 bit ALU REG operations */
case (BPF_ALU | BPF_ADD | BPF_X):
BPF_OP_ALU_REG(reg, ins, +, uint32_t);
break;
case (BPF_ALU | BPF_SUB | BPF_X):
BPF_OP_ALU_REG(reg, ins, -, uint32_t);
break;
case (BPF_ALU | BPF_AND | BPF_X):
BPF_OP_ALU_REG(reg, ins, &, uint32_t);
break;
case (BPF_ALU | BPF_OR | BPF_X):
BPF_OP_ALU_REG(reg, ins, |, uint32_t);
break;
case (BPF_ALU | BPF_LSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, <<, uint32_t);
break;
case (BPF_ALU | BPF_RSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, uint32_t);
break;
case (BPF_ALU | BPF_XOR | BPF_X):
BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
break;
case (BPF_ALU | BPF_MUL | BPF_X):
BPF_OP_ALU_REG(reg, ins, *, uint32_t);
break;
case (BPF_ALU | BPF_DIV | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
BPF_OP_ALU_REG(reg, ins, /, uint32_t);
break;
case (BPF_ALU | BPF_MOD | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
BPF_OP_ALU_REG(reg, ins, %, uint32_t);
break;
case (BPF_ALU | EBPF_MOV | BPF_X):
EBPF_MOV_ALU_REG(reg, ins, uint32_t);
break;
case (BPF_ALU | BPF_NEG):
BPF_NEG_ALU(reg, ins, uint32_t);
break;
case (BPF_ALU | EBPF_END | EBPF_TO_BE):
bpf_alu_be(reg, ins);
break;
case (BPF_ALU | EBPF_END | EBPF_TO_LE):
bpf_alu_le(reg, ins);
break;
/* 64 bit ALU IMM operations */
case (EBPF_ALU64 | BPF_ADD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, +, uint64_t);
break;
case (EBPF_ALU64 | BPF_SUB | BPF_K):
BPF_OP_ALU_IMM(reg, ins, -, uint64_t);
break;
case (EBPF_ALU64 | BPF_AND | BPF_K):
BPF_OP_ALU_IMM(reg, ins, &, uint64_t);
break;
case (EBPF_ALU64 | BPF_OR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, |, uint64_t);
break;
case (EBPF_ALU64 | BPF_LSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);
break;
case (EBPF_ALU64 | BPF_RSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);
break;
case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, int64_t);
break;
case (EBPF_ALU64 | BPF_XOR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
break;
case (EBPF_ALU64 | BPF_MUL | BPF_K):
BPF_OP_ALU_IMM(reg, ins, *, uint64_t);
break;
case (EBPF_ALU64 | BPF_DIV | BPF_K):
BPF_OP_ALU_IMM(reg, ins, /, uint64_t);
break;
case (EBPF_ALU64 | BPF_MOD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, %, uint64_t);
break;
case (EBPF_ALU64 | EBPF_MOV | BPF_K):
EBPF_MOV_ALU_IMM(reg, ins, uint64_t);
break;
/* 64 bit ALU REG operations */
case (EBPF_ALU64 | BPF_ADD | BPF_X):
BPF_OP_ALU_REG(reg, ins, +, uint64_t);
break;
case (EBPF_ALU64 | BPF_SUB | BPF_X):
BPF_OP_ALU_REG(reg, ins, -, uint64_t);
break;
case (EBPF_ALU64 | BPF_AND | BPF_X):
BPF_OP_ALU_REG(reg, ins, &, uint64_t);
break;
case (EBPF_ALU64 | BPF_OR | BPF_X):
BPF_OP_ALU_REG(reg, ins, |, uint64_t);
break;
case (EBPF_ALU64 | BPF_LSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, <<, uint64_t);
break;
case (EBPF_ALU64 | BPF_RSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, uint64_t);
break;
case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, int64_t);
break;
case (EBPF_ALU64 | BPF_XOR | BPF_X):
BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
break;
case (EBPF_ALU64 | BPF_MUL | BPF_X):
BPF_OP_ALU_REG(reg, ins, *, uint64_t);
break;
case (EBPF_ALU64 | BPF_DIV | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
BPF_OP_ALU_REG(reg, ins, /, uint64_t);
break;
case (EBPF_ALU64 | BPF_MOD | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
BPF_OP_ALU_REG(reg, ins, %, uint64_t);
break;
case (EBPF_ALU64 | EBPF_MOV | BPF_X):
EBPF_MOV_ALU_REG(reg, ins, uint64_t);
break;
case (EBPF_ALU64 | BPF_NEG):
BPF_NEG_ALU(reg, ins, uint64_t);
break;
/* load instructions */
case (BPF_LDX | BPF_MEM | BPF_B):
BPF_LD_REG(reg, ins, uint8_t);
break;
case (BPF_LDX | BPF_MEM | BPF_H):
BPF_LD_REG(reg, ins, uint16_t);
break;
case (BPF_LDX | BPF_MEM | BPF_W):
BPF_LD_REG(reg, ins, uint32_t);
break;
case (BPF_LDX | BPF_MEM | EBPF_DW):
BPF_LD_REG(reg, ins, uint64_t);
break;
/* load 64 bit immediate value */
case (BPF_LD | BPF_IMM | EBPF_DW):
reg[ins->dst_reg] = (uint32_t)ins[0].imm |
(uint64_t)(uint32_t)ins[1].imm << 32;
ins++;
break;
/* load absolute instructions */
case (BPF_LD | BPF_ABS | BPF_B):
BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);
break;
case (BPF_LD | BPF_ABS | BPF_H):
BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
break;
case (BPF_LD | BPF_ABS | BPF_W):
BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
break;
/* load indirect instructions */
case (BPF_LD | BPF_IND | BPF_B):
BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);
break;
case (BPF_LD | BPF_IND | BPF_H):
BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
break;
case (BPF_LD | BPF_IND | BPF_W):
BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
break;
/* store instructions */
case (BPF_STX | BPF_MEM | BPF_B):
BPF_ST_REG(reg, ins, uint8_t);
break;
case (BPF_STX | BPF_MEM | BPF_H):
BPF_ST_REG(reg, ins, uint16_t);
break;
case (BPF_STX | BPF_MEM | BPF_W):
BPF_ST_REG(reg, ins, uint32_t);
break;
case (BPF_STX | BPF_MEM | EBPF_DW):
BPF_ST_REG(reg, ins, uint64_t);
break;
case (BPF_ST | BPF_MEM | BPF_B):
BPF_ST_IMM(reg, ins, uint8_t);
break;
case (BPF_ST | BPF_MEM | BPF_H):
BPF_ST_IMM(reg, ins, uint16_t);
break;
case (BPF_ST | BPF_MEM | BPF_W):
BPF_ST_IMM(reg, ins, uint32_t);
break;
case (BPF_ST | BPF_MEM | EBPF_DW):
BPF_ST_IMM(reg, ins, uint64_t);
break;
/* atomic add instructions */
case (BPF_STX | EBPF_XADD | BPF_W):
BPF_ST_XADD_REG(reg, ins, 32);
break;
case (BPF_STX | EBPF_XADD | EBPF_DW):
BPF_ST_XADD_REG(reg, ins, 64);
break;
/* jump instructions */
case (BPF_JMP | BPF_JA):
BPF_JMP_UNC(ins);
break;
/* jump IMM instructions */
case (BPF_JMP | BPF_JEQ | BPF_K):
BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);
break;
case (BPF_JMP | EBPF_JNE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);
break;
case (BPF_JMP | BPF_JGT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >, uint64_t);
break;
case (BPF_JMP | EBPF_JLT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <, uint64_t);
break;
case (BPF_JMP | BPF_JGE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);
break;
case (BPF_JMP | EBPF_JLE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);
break;
case (BPF_JMP | EBPF_JSGT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >, int64_t);
break;
case (BPF_JMP | EBPF_JSLT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <, int64_t);
break;
case (BPF_JMP | EBPF_JSGE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >=, int64_t);
break;
case (BPF_JMP | EBPF_JSLE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <=, int64_t);
break;
case (BPF_JMP | BPF_JSET | BPF_K):
BPF_JMP_CND_IMM(reg, ins, &, uint64_t);
break;
/* jump REG instructions */
case (BPF_JMP | BPF_JEQ | BPF_X):
BPF_JMP_CND_REG(reg, ins, ==, uint64_t);
break;
case (BPF_JMP | EBPF_JNE | BPF_X):
BPF_JMP_CND_REG(reg, ins, !=, uint64_t);
break;
case (BPF_JMP | BPF_JGT | BPF_X):
BPF_JMP_CND_REG(reg, ins, >, uint64_t);
break;
case (BPF_JMP | EBPF_JLT | BPF_X):
BPF_JMP_CND_REG(reg, ins, <, uint64_t);
break;
case (BPF_JMP | BPF_JGE | BPF_X):
BPF_JMP_CND_REG(reg, ins, >=, uint64_t);
break;
case (BPF_JMP | EBPF_JLE | BPF_X):
BPF_JMP_CND_REG(reg, ins, <=, uint64_t);
break;
case (BPF_JMP | EBPF_JSGT | BPF_X):
BPF_JMP_CND_REG(reg, ins, >, int64_t);
break;
case (BPF_JMP | EBPF_JSLT | BPF_X):
BPF_JMP_CND_REG(reg, ins, <, int64_t);
break;
case (BPF_JMP | EBPF_JSGE | BPF_X):
BPF_JMP_CND_REG(reg, ins, >=, int64_t);
break;
case (BPF_JMP | EBPF_JSLE | BPF_X):
BPF_JMP_CND_REG(reg, ins, <=, int64_t);
break;
case (BPF_JMP | BPF_JSET | BPF_X):
BPF_JMP_CND_REG(reg, ins, &, uint64_t);
break;
/* call instructions */
case (BPF_JMP | EBPF_CALL):
reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
reg[EBPF_REG_1], reg[EBPF_REG_2],
reg[EBPF_REG_3], reg[EBPF_REG_4],
reg[EBPF_REG_5]);
break;
/* return instruction */
case (BPF_JMP | EBPF_EXIT):
return reg[EBPF_REG_0];
default:
RTE_BPF_LOG(ERR,
"%s(%p): invalid opcode %#x at pc: %#zx;\n",
__func__, bpf, ins->code,
(uintptr_t)ins - (uintptr_t)bpf->prm.ins);
return 0;
}
}
/* should never be reached */
RTE_VERIFY(0);
return 0;
}
uint32_t
rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
uint32_t num)
{
uint32_t i;
uint64_t reg[EBPF_REG_NUM];
uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
for (i = 0; i != num; i++) {
reg[EBPF_REG_1] = (uintptr_t)ctx[i];
reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
rc[i] = bpf_exec(bpf, reg);
}
return i;
}
uint64_t
rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
{
uint64_t rc;
rte_bpf_exec_burst(bpf, &ctx, &rc, 1);
return rc;
}
Reference in New Issue View Git Blame Copy Permalink