d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
e4f9eab7d9
numam-dpdk/lib/librte_bpf/bpf_exec.c
Konstantin Ananyev b901d92836 bpf: support packet data load instructions 
To fill the gap with linux kernel eBPF implementation,
add support for two non-generic instructions:
(BPF_ABS | <size> | BPF_LD) and (BPF_IND | <size> | BPF_LD)
which are used to access packet data.
These instructions can only be used when BPF context is a pointer
to 'struct rte_mbuf' (i.e: RTE_BPF_ARG_PTR_MBUF type).

Signed-off-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
2020-06-24 23:42:04 +02:00

511 lines
14 KiB
C
Raw Blame History

/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2018 Intel Corporation
*/
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#include <errno.h>
#include <stdint.h>
#include <inttypes.h>
#include <rte_common.h>
#include <rte_log.h>
#include <rte_debug.h>
#include <rte_memory.h>
#include <rte_eal.h>
#include <rte_byteorder.h>
#include "bpf_impl.h"
#define BPF_JMP_UNC(ins) ((ins) += (ins)->off)
#define BPF_JMP_CND_REG(reg, ins, op, type) \
((ins) += \
((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
(ins)->off : 0)
#define BPF_JMP_CND_IMM(reg, ins, op, type) \
((ins) += \
((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
(ins)->off : 0)
#define BPF_NEG_ALU(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
#define EBPF_MOV_ALU_REG(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
#define BPF_OP_ALU_REG(reg, ins, op, type) \
((reg)[(ins)->dst_reg] = \
(type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
#define EBPF_MOV_ALU_IMM(reg, ins, type) \
((reg)[(ins)->dst_reg] = (type)(ins)->imm)
#define BPF_OP_ALU_IMM(reg, ins, op, type) \
((reg)[(ins)->dst_reg] = \
(type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
#define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \
if ((type)(reg)[(ins)->src_reg] == 0) { \
RTE_BPF_LOG(ERR, \
"%s(%p): division by 0 at pc: %#zx;\n", \
__func__, bpf, \
(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
return 0; \
} \
} while (0)
#define BPF_LD_REG(reg, ins, type) \
((reg)[(ins)->dst_reg] = \
*(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
#define BPF_ST_IMM(reg, ins, type) \
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(ins)->imm)
#define BPF_ST_REG(reg, ins, type) \
(*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
(type)(reg)[(ins)->src_reg])
#define BPF_ST_XADD_REG(reg, ins, tp) \
(rte_atomic##tp##_add((rte_atomic##tp##_t *) \
(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
reg[ins->src_reg]))
/* BPF_LD | BPF_ABS/BPF_IND */
#define NOP(x) (x)
#define BPF_LD_ABS(bpf, reg, ins, type, op) do { \
const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
if (p == NULL) \
return 0; \
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
#define BPF_LD_IND(bpf, reg, ins, type, op) do { \
uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
if (p == NULL) \
return 0; \
reg[EBPF_REG_0] = op(p[0]); \
} while (0)
static inline void
bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
{
uint64_t *v;
v = reg + ins->dst_reg;
switch (ins->imm) {
case 16:
*v = rte_cpu_to_be_16(*v);
break;
case 32:
*v = rte_cpu_to_be_32(*v);
break;
case 64:
*v = rte_cpu_to_be_64(*v);
break;
}
}
static inline void
bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
{
uint64_t *v;
v = reg + ins->dst_reg;
switch (ins->imm) {
case 16:
*v = rte_cpu_to_le_16(*v);
break;
case 32:
*v = rte_cpu_to_le_32(*v);
break;
case 64:
*v = rte_cpu_to_le_64(*v);
break;
}
}
static inline const void *
bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
const struct ebpf_insn *ins, uint32_t off, uint32_t len)
{
const struct rte_mbuf *mb;
const void *p;
mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6];
p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0);
if (p == NULL)
RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
"load beyond packet boundary at pc: %#zx;\n",
__func__, bpf, mb, off, len,
(uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
return p;
}
static inline uint64_t
bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
{
const struct ebpf_insn *ins;
for (ins = bpf->prm.ins; ; ins++) {
switch (ins->code) {
/* 32 bit ALU IMM operations */
case (BPF_ALU | BPF_ADD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, +, uint32_t);
break;
case (BPF_ALU | BPF_SUB | BPF_K):
BPF_OP_ALU_IMM(reg, ins, -, uint32_t);
break;
case (BPF_ALU | BPF_AND | BPF_K):
BPF_OP_ALU_IMM(reg, ins, &, uint32_t);
break;
case (BPF_ALU | BPF_OR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, |, uint32_t);
break;
case (BPF_ALU | BPF_LSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);
break;
case (BPF_ALU | BPF_RSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);
break;
case (BPF_ALU | BPF_XOR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
break;
case (BPF_ALU | BPF_MUL | BPF_K):
BPF_OP_ALU_IMM(reg, ins, *, uint32_t);
break;
case (BPF_ALU | BPF_DIV | BPF_K):
BPF_OP_ALU_IMM(reg, ins, /, uint32_t);
break;
case (BPF_ALU | BPF_MOD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, %, uint32_t);
break;
case (BPF_ALU | EBPF_MOV | BPF_K):
EBPF_MOV_ALU_IMM(reg, ins, uint32_t);
break;
/* 32 bit ALU REG operations */
case (BPF_ALU | BPF_ADD | BPF_X):
BPF_OP_ALU_REG(reg, ins, +, uint32_t);
break;
case (BPF_ALU | BPF_SUB | BPF_X):
BPF_OP_ALU_REG(reg, ins, -, uint32_t);
break;
case (BPF_ALU | BPF_AND | BPF_X):
BPF_OP_ALU_REG(reg, ins, &, uint32_t);
break;
case (BPF_ALU | BPF_OR | BPF_X):
BPF_OP_ALU_REG(reg, ins, |, uint32_t);
break;
case (BPF_ALU | BPF_LSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, <<, uint32_t);
break;
case (BPF_ALU | BPF_RSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, uint32_t);
break;
case (BPF_ALU | BPF_XOR | BPF_X):
BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
break;
case (BPF_ALU | BPF_MUL | BPF_X):
BPF_OP_ALU_REG(reg, ins, *, uint32_t);
break;
case (BPF_ALU | BPF_DIV | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
BPF_OP_ALU_REG(reg, ins, /, uint32_t);
break;
case (BPF_ALU | BPF_MOD | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
BPF_OP_ALU_REG(reg, ins, %, uint32_t);
break;
case (BPF_ALU | EBPF_MOV | BPF_X):
EBPF_MOV_ALU_REG(reg, ins, uint32_t);
break;
case (BPF_ALU | BPF_NEG):
BPF_NEG_ALU(reg, ins, uint32_t);
break;
case (BPF_ALU | EBPF_END | EBPF_TO_BE):
bpf_alu_be(reg, ins);
break;
case (BPF_ALU | EBPF_END | EBPF_TO_LE):
bpf_alu_le(reg, ins);
break;
/* 64 bit ALU IMM operations */
case (EBPF_ALU64 | BPF_ADD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, +, uint64_t);
break;
case (EBPF_ALU64 | BPF_SUB | BPF_K):
BPF_OP_ALU_IMM(reg, ins, -, uint64_t);
break;
case (EBPF_ALU64 | BPF_AND | BPF_K):
BPF_OP_ALU_IMM(reg, ins, &, uint64_t);
break;
case (EBPF_ALU64 | BPF_OR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, |, uint64_t);
break;
case (EBPF_ALU64 | BPF_LSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);
break;
case (EBPF_ALU64 | BPF_RSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);
break;
case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
BPF_OP_ALU_IMM(reg, ins, >>, int64_t);
break;
case (EBPF_ALU64 | BPF_XOR | BPF_K):
BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
break;
case (EBPF_ALU64 | BPF_MUL | BPF_K):
BPF_OP_ALU_IMM(reg, ins, *, uint64_t);
break;
case (EBPF_ALU64 | BPF_DIV | BPF_K):
BPF_OP_ALU_IMM(reg, ins, /, uint64_t);
break;
case (EBPF_ALU64 | BPF_MOD | BPF_K):
BPF_OP_ALU_IMM(reg, ins, %, uint64_t);
break;
case (EBPF_ALU64 | EBPF_MOV | BPF_K):
EBPF_MOV_ALU_IMM(reg, ins, uint64_t);
break;
/* 64 bit ALU REG operations */
case (EBPF_ALU64 | BPF_ADD | BPF_X):
BPF_OP_ALU_REG(reg, ins, +, uint64_t);
break;
case (EBPF_ALU64 | BPF_SUB | BPF_X):
BPF_OP_ALU_REG(reg, ins, -, uint64_t);
break;
case (EBPF_ALU64 | BPF_AND | BPF_X):
BPF_OP_ALU_REG(reg, ins, &, uint64_t);
break;
case (EBPF_ALU64 | BPF_OR | BPF_X):
BPF_OP_ALU_REG(reg, ins, |, uint64_t);
break;
case (EBPF_ALU64 | BPF_LSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, <<, uint64_t);
break;
case (EBPF_ALU64 | BPF_RSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, uint64_t);
break;
case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
BPF_OP_ALU_REG(reg, ins, >>, int64_t);
break;
case (EBPF_ALU64 | BPF_XOR | BPF_X):
BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
break;
case (EBPF_ALU64 | BPF_MUL | BPF_X):
BPF_OP_ALU_REG(reg, ins, *, uint64_t);
break;
case (EBPF_ALU64 | BPF_DIV | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
BPF_OP_ALU_REG(reg, ins, /, uint64_t);
break;
case (EBPF_ALU64 | BPF_MOD | BPF_X):
BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
BPF_OP_ALU_REG(reg, ins, %, uint64_t);
break;
case (EBPF_ALU64 | EBPF_MOV | BPF_X):
EBPF_MOV_ALU_REG(reg, ins, uint64_t);
break;
case (EBPF_ALU64 | BPF_NEG):
BPF_NEG_ALU(reg, ins, uint64_t);
break;
/* load instructions */
case (BPF_LDX | BPF_MEM | BPF_B):
BPF_LD_REG(reg, ins, uint8_t);
break;
case (BPF_LDX | BPF_MEM | BPF_H):
BPF_LD_REG(reg, ins, uint16_t);
break;
case (BPF_LDX | BPF_MEM | BPF_W):
BPF_LD_REG(reg, ins, uint32_t);
break;
case (BPF_LDX | BPF_MEM | EBPF_DW):
BPF_LD_REG(reg, ins, uint64_t);
break;
/* load 64 bit immediate value */
case (BPF_LD | BPF_IMM | EBPF_DW):
reg[ins->dst_reg] = (uint32_t)ins[0].imm |
(uint64_t)(uint32_t)ins[1].imm << 32;
ins++;
break;
/* load absolute instructions */
case (BPF_LD | BPF_ABS | BPF_B):
BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);
break;
case (BPF_LD | BPF_ABS | BPF_H):
BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
break;
case (BPF_LD | BPF_ABS | BPF_W):
BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
break;
/* load indirect instructions */
case (BPF_LD | BPF_IND | BPF_B):
BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);
break;
case (BPF_LD | BPF_IND | BPF_H):
BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
break;
case (BPF_LD | BPF_IND | BPF_W):
BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
break;
/* store instructions */
case (BPF_STX | BPF_MEM | BPF_B):
BPF_ST_REG(reg, ins, uint8_t);
break;
case (BPF_STX | BPF_MEM | BPF_H):
BPF_ST_REG(reg, ins, uint16_t);
break;
case (BPF_STX | BPF_MEM | BPF_W):
BPF_ST_REG(reg, ins, uint32_t);
break;
case (BPF_STX | BPF_MEM | EBPF_DW):
BPF_ST_REG(reg, ins, uint64_t);
break;
case (BPF_ST | BPF_MEM | BPF_B):
BPF_ST_IMM(reg, ins, uint8_t);
break;
case (BPF_ST | BPF_MEM | BPF_H):
BPF_ST_IMM(reg, ins, uint16_t);
break;
case (BPF_ST | BPF_MEM | BPF_W):
BPF_ST_IMM(reg, ins, uint32_t);
break;
case (BPF_ST | BPF_MEM | EBPF_DW):
BPF_ST_IMM(reg, ins, uint64_t);
break;
/* atomic add instructions */
case (BPF_STX | EBPF_XADD | BPF_W):
BPF_ST_XADD_REG(reg, ins, 32);
break;
case (BPF_STX | EBPF_XADD | EBPF_DW):
BPF_ST_XADD_REG(reg, ins, 64);
break;
/* jump instructions */
case (BPF_JMP | BPF_JA):
BPF_JMP_UNC(ins);
break;
/* jump IMM instructions */
case (BPF_JMP | BPF_JEQ | BPF_K):
BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);
break;
case (BPF_JMP | EBPF_JNE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);
break;
case (BPF_JMP | BPF_JGT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >, uint64_t);
break;
case (BPF_JMP | EBPF_JLT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <, uint64_t);
break;
case (BPF_JMP | BPF_JGE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);
break;
case (BPF_JMP | EBPF_JLE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);
break;
case (BPF_JMP | EBPF_JSGT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >, int64_t);
break;
case (BPF_JMP | EBPF_JSLT | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <, int64_t);
break;
case (BPF_JMP | EBPF_JSGE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, >=, int64_t);
break;
case (BPF_JMP | EBPF_JSLE | BPF_K):
BPF_JMP_CND_IMM(reg, ins, <=, int64_t);
break;
case (BPF_JMP | BPF_JSET | BPF_K):
BPF_JMP_CND_IMM(reg, ins, &, uint64_t);
break;
/* jump REG instructions */
case (BPF_JMP | BPF_JEQ | BPF_X):
BPF_JMP_CND_REG(reg, ins, ==, uint64_t);
break;
case (BPF_JMP | EBPF_JNE | BPF_X):
BPF_JMP_CND_REG(reg, ins, !=, uint64_t);
break;
case (BPF_JMP | BPF_JGT | BPF_X):
BPF_JMP_CND_REG(reg, ins, >, uint64_t);
break;
case (BPF_JMP | EBPF_JLT | BPF_X):
BPF_JMP_CND_REG(reg, ins, <, uint64_t);
break;
case (BPF_JMP | BPF_JGE | BPF_X):
BPF_JMP_CND_REG(reg, ins, >=, uint64_t);
break;
case (BPF_JMP | EBPF_JLE | BPF_X):
BPF_JMP_CND_REG(reg, ins, <=, uint64_t);
break;
case (BPF_JMP | EBPF_JSGT | BPF_X):
BPF_JMP_CND_REG(reg, ins, >, int64_t);
break;
case (BPF_JMP | EBPF_JSLT | BPF_X):
BPF_JMP_CND_REG(reg, ins, <, int64_t);
break;
case (BPF_JMP | EBPF_JSGE | BPF_X):
BPF_JMP_CND_REG(reg, ins, >=, int64_t);
break;
case (BPF_JMP | EBPF_JSLE | BPF_X):
BPF_JMP_CND_REG(reg, ins, <=, int64_t);
break;
case (BPF_JMP | BPF_JSET | BPF_X):
BPF_JMP_CND_REG(reg, ins, &, uint64_t);
break;
/* call instructions */
case (BPF_JMP | EBPF_CALL):
reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
reg[EBPF_REG_1], reg[EBPF_REG_2],
reg[EBPF_REG_3], reg[EBPF_REG_4],
reg[EBPF_REG_5]);
break;
/* return instruction */
case (BPF_JMP | EBPF_EXIT):
return reg[EBPF_REG_0];
default:
RTE_BPF_LOG(ERR,
"%s(%p): invalid opcode %#x at pc: %#zx;\n",
__func__, bpf, ins->code,
(uintptr_t)ins - (uintptr_t)bpf->prm.ins);
return 0;
}
}
/* should never be reached */
RTE_VERIFY(0);
return 0;
}
uint32_t
rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
uint32_t num)
{
uint32_t i;
uint64_t reg[EBPF_REG_NUM];
uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
for (i = 0; i != num; i++) {
reg[EBPF_REG_1] = (uintptr_t)ctx[i];
reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
rc[i] = bpf_exec(bpf, reg);
}
return i;
}
uint64_t
rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
{
uint64_t rc;
rte_bpf_exec_burst(bpf, &ctx, &rc, 1);
return rc;
}
Reference in New Issue View Git Blame Copy Permalink