d/numam-dpdk
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

bpf/arm: add atomic-exchange-and-add operation

Browse Source 
Implement XADD eBPF instruction using STADD arm64 instruction.
If the given platform does not have atomics support,
use LDXR and STXR pair for critical section instead of STADD.

Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>
This commit is contained in:
Jerin Jacob 2019-09-03 16:29:37 +05:30 committed by Thomas Monjalon
parent e00906bdc7
commit 2acfae37f6
1 changed files with 84 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
lib/librte_bpf/bpf_jit_arm64.c
View File

@ -837,6 +837,83 @@ emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
emit_b(ctx, jump_to_epilogue);
}
static void
emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
{
uint32_t insn;
insn = 0xb820001f;
insn |= (!!is64) << 30;
insn |= rs << 16;
insn |= rn << 5;
emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
}
static void
emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
{
uint32_t insn;
insn = 0x885f7c00;
insn |= (!!is64) << 30;
insn |= rn << 5;
insn |= rt;
emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
}
static void
emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
uint8_t rn)
{
uint32_t insn;
insn = 0x88007c00;
insn |= (!!is64) << 30;
insn |= rs << 16;
insn |= rn << 5;
insn |= rt;
emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
}
static int
has_atomics(void)
{
int rc = 0;
#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
rc = 1;
#endif
return rc;
}
static void
emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
{
bool is64 = (BPF_SIZE(op) == EBPF_DW);
uint8_t rn;
if (off) {
emit_mov_imm(ctx, 1, tmp1, off);
emit_add(ctx, 1, tmp1, dst);
rn = tmp1;
} else {
rn = dst;
}
if (has_atomics()) {
emit_stadd(ctx, is64, src, rn);
} else {
emit_ldxr(ctx, is64, tmp2, rn);
emit_add(ctx, is64, tmp2, src);
emit_stxr(ctx, is64, tmp3, tmp2, rn);
emit_cbnz(ctx, is64, tmp3, -3);
}
}
static void
check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
@ -863,7 +940,7 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
static int
emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
{
uint8_t op, dst, src, tmp1, tmp2;
uint8_t op, dst, src, tmp1, tmp2, tmp3;
const struct ebpf_insn *ins;
uint64_t u64;
int16_t off;
@ -878,6 +955,7 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
emit_prologue(ctx);
@ -1067,6 +1145,11 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
emit_mov_imm(ctx, 1, tmp2, off);
emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
break;
/* STX XADD: lock *(size *)(dst + off) += src */
case (BPF_STX | EBPF_XADD | BPF_W):
case (BPF_STX | EBPF_XADD | EBPF_DW):
emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
break;
/* Return r0 */
case (BPF_JMP | EBPF_EXIT):
emit_epilogue(ctx);