- Allocate scratch memory on stack instead of pre-allocating it with
the filter as we do from bpf_filter()[1]. - Revert experimental use of contigmalloc(9)/contigfree(9). It has no performance benefit over malloc(9)/free(9)[2]. Requested by: rwatson[1] Pointed out by: rwatson, jhb, alc[2]
This commit is contained in:
parent
0c8e3b18f0
commit
5ecf77367c
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=199603
@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <amd64/amd64/bpf_jit_machdep.h>
|
||||
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
|
||||
|
||||
/*
|
||||
* emit routine to update the jump table
|
||||
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
|
||||
* Function that does the real stuff
|
||||
*/
|
||||
bpf_filter_func
|
||||
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
|
||||
{
|
||||
bpf_bin_stream stream;
|
||||
struct bpf_insn *ins;
|
||||
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
|
||||
/* Allocate the reference table for the jumps */
|
||||
#ifdef _KERNEL
|
||||
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
|
||||
M_BPFJIT, M_NOWAIT);
|
||||
stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
|
||||
#else
|
||||
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
|
||||
stream.refs = malloc((nins + 1) * sizeof(u_int));
|
||||
#endif
|
||||
if (stream.refs == NULL)
|
||||
return (NULL);
|
||||
@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
ins = prog;
|
||||
|
||||
/* create the procedure header */
|
||||
MOVrq2(RBX, R8);
|
||||
MOVrq(RDI, RBX);
|
||||
PUSH(RBP);
|
||||
MOVrq(RSP, RBP);
|
||||
SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
|
||||
MOVrq2(RDI, R8);
|
||||
MOVrd2(ESI, R9D);
|
||||
MOVrd(EDX, EDI);
|
||||
|
||||
@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
|
||||
case BPF_RET|BPF_K:
|
||||
MOVid(ins->k, EAX);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
LEAVE_RET();
|
||||
break;
|
||||
|
||||
case BPF_RET|BPF_A:
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
LEAVE_RET();
|
||||
break;
|
||||
|
||||
case BPF_LD|BPF_W|BPF_ABS:
|
||||
@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
MOVrd(EDI, ECX);
|
||||
SUBrd(ESI, ECX);
|
||||
CMPid(sizeof(int32_t), ECX);
|
||||
JAEb(6);
|
||||
JAEb(4);
|
||||
ZEROrd(EAX);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
MOVobd(RBX, RSI, EAX);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobd(RCX, RSI, EAX);
|
||||
BSWAP(EAX);
|
||||
break;
|
||||
|
||||
@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
MOVrd(EDI, ECX);
|
||||
SUBrd(ESI, ECX);
|
||||
CMPid(sizeof(int16_t), ECX);
|
||||
JAEb(4);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
MOVobw(RBX, RSI, AX);
|
||||
JAEb(2);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobw(RCX, RSI, AX);
|
||||
SWAP_AX();
|
||||
break;
|
||||
|
||||
@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
ZEROrd(EAX);
|
||||
MOVid(ins->k, ESI);
|
||||
CMPrd(EDI, ESI);
|
||||
JBb(4);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
MOVobb(RBX, RSI, AL);
|
||||
JBb(2);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobb(RCX, RSI, AL);
|
||||
break;
|
||||
|
||||
case BPF_LD|BPF_W|BPF_LEN:
|
||||
@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
MOVrd(EDI, ECX);
|
||||
SUBrd(ESI, ECX);
|
||||
CMPid(sizeof(int32_t), ECX);
|
||||
JAEb(6);
|
||||
JAEb(4);
|
||||
ZEROrd(EAX);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
MOVobd(RBX, RSI, EAX);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobd(RCX, RSI, EAX);
|
||||
BSWAP(EAX);
|
||||
break;
|
||||
|
||||
@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
MOVrd(EDI, ECX);
|
||||
SUBrd(ESI, ECX);
|
||||
CMPid(sizeof(int16_t), ECX);
|
||||
JAEb(4);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
MOVobw(RBX, RSI, AX);
|
||||
JAEb(2);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobw(RCX, RSI, AX);
|
||||
SWAP_AX();
|
||||
break;
|
||||
|
||||
@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
MOVrd(EDI, ECX);
|
||||
SUBrd(EDX, ECX);
|
||||
CMPrd(ESI, ECX);
|
||||
JAb(4);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
JAb(2);
|
||||
LEAVE_RET();
|
||||
MOVrq3(R8, RCX);
|
||||
ADDrd(EDX, ESI);
|
||||
MOVobb(RBX, RSI, AL);
|
||||
MOVobb(RCX, RSI, AL);
|
||||
break;
|
||||
|
||||
case BPF_LDX|BPF_MSH|BPF_B:
|
||||
MOVid(ins->k, ESI);
|
||||
CMPrd(EDI, ESI);
|
||||
JBb(6);
|
||||
JBb(4);
|
||||
ZEROrd(EAX);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
LEAVE_RET();
|
||||
ZEROrd(EDX);
|
||||
MOVobb(RBX, RSI, DL);
|
||||
MOVrq3(R8, RCX);
|
||||
MOVobb(RCX, RSI, DL);
|
||||
ANDib(0x0f, DL);
|
||||
SHLib(2, EDX);
|
||||
break;
|
||||
@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
break;
|
||||
|
||||
case BPF_LD|BPF_MEM:
|
||||
MOViq((uintptr_t)mem, RCX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVobd(RCX, RSI, EAX);
|
||||
MOVid(ins->k * sizeof(uint32_t), ESI);
|
||||
MOVobd(RSP, RSI, EAX);
|
||||
break;
|
||||
|
||||
case BPF_LDX|BPF_MEM:
|
||||
MOViq((uintptr_t)mem, RCX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVobd(RCX, RSI, EDX);
|
||||
MOVid(ins->k * sizeof(uint32_t), ESI);
|
||||
MOVobd(RSP, RSI, EDX);
|
||||
break;
|
||||
|
||||
case BPF_ST:
|
||||
@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
* be optimized if the previous instruction
|
||||
* was already of this type
|
||||
*/
|
||||
MOViq((uintptr_t)mem, RCX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVomd(EAX, RCX, RSI);
|
||||
MOVid(ins->k * sizeof(uint32_t), ESI);
|
||||
MOVomd(EAX, RSP, RSI);
|
||||
break;
|
||||
|
||||
case BPF_STX:
|
||||
MOViq((uintptr_t)mem, RCX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVomd(EDX, RCX, RSI);
|
||||
MOVid(ins->k * sizeof(uint32_t), ESI);
|
||||
MOVomd(EDX, RSP, RSI);
|
||||
break;
|
||||
|
||||
case BPF_JMP|BPF_JA:
|
||||
@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
|
||||
case BPF_ALU|BPF_DIV|BPF_X:
|
||||
TESTrd(EDX, EDX);
|
||||
JNEb(6);
|
||||
JNEb(4);
|
||||
ZEROrd(EAX);
|
||||
MOVrq3(R8, RBX);
|
||||
RET();
|
||||
LEAVE_RET();
|
||||
MOVrd(EDX, ECX);
|
||||
ZEROrd(EDX);
|
||||
DIVrd(ECX);
|
||||
@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
}
|
||||
|
||||
#ifdef _KERNEL
|
||||
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
|
||||
M_NOWAIT, 0, ~0UL, 16, 0);
|
||||
stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
|
||||
if (stream.ibuf == NULL)
|
||||
break;
|
||||
#else
|
||||
stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
|
||||
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
|
||||
stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
|
||||
MAP_ANON, -1, 0);
|
||||
if (stream.ibuf == MAP_FAILED) {
|
||||
stream.ibuf = NULL;
|
||||
break;
|
||||
|
@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
|
||||
emitm(&stream, 0xc486, 2); \
|
||||
} while (0)
|
||||
|
||||
/* ret */
|
||||
#define RET() do { \
|
||||
emitm(&stream, 0xc3, 1); \
|
||||
/* pushq r64 */
|
||||
#define PUSH(r64) do { \
|
||||
emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \
|
||||
} while (0)
|
||||
|
||||
/* leave/ret */
|
||||
#define LEAVE_RET() do { \
|
||||
emitm(&stream, 0xc3c9, 2); \
|
||||
} while (0)
|
||||
|
||||
/* addl sr32,dr32 */
|
||||
@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
|
||||
emitm(&stream, i32, 4); \
|
||||
} while (0)
|
||||
|
||||
/* subq i8,r64 */
|
||||
#define SUBib(i8, r64) do { \
|
||||
emitm(&stream, 0x8348, 2); \
|
||||
emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \
|
||||
emitm(&stream, i8, 1); \
|
||||
} while (0)
|
||||
|
||||
/* mull r32 */
|
||||
#define MULrd(r32) do { \
|
||||
emitm(&stream, 0xf7, 1); \
|
||||
|
@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
|
||||
|
||||
#include <i386/i386/bpf_jit_machdep.h>
|
||||
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
|
||||
|
||||
/*
|
||||
* emit routine to update the jump table
|
||||
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
|
||||
* Function that does the real stuff
|
||||
*/
|
||||
bpf_filter_func
|
||||
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
|
||||
{
|
||||
bpf_bin_stream stream;
|
||||
struct bpf_insn *ins;
|
||||
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
|
||||
/* Allocate the reference table for the jumps */
|
||||
#ifdef _KERNEL
|
||||
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
|
||||
M_BPFJIT, M_NOWAIT);
|
||||
stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
|
||||
#else
|
||||
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
|
||||
stream.refs = malloc((nins + 1) * sizeof(u_int));
|
||||
#endif
|
||||
if (stream.refs == NULL)
|
||||
return (NULL);
|
||||
@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
/* create the procedure header */
|
||||
PUSH(EBP);
|
||||
MOVrd(ESP, EBP);
|
||||
SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
|
||||
PUSH(EDI);
|
||||
PUSH(ESI);
|
||||
PUSH(EBX);
|
||||
@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
break;
|
||||
|
||||
case BPF_LD|BPF_MEM:
|
||||
MOVid((uintptr_t)mem, ECX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVrd(EBP, ECX);
|
||||
MOVid(((int)ins->k - BPF_MEMWORDS) *
|
||||
sizeof(uint32_t), ESI);
|
||||
MOVobd(ECX, ESI, EAX);
|
||||
break;
|
||||
|
||||
case BPF_LDX|BPF_MEM:
|
||||
MOVid((uintptr_t)mem, ECX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVrd(EBP, ECX);
|
||||
MOVid(((int)ins->k - BPF_MEMWORDS) *
|
||||
sizeof(uint32_t), ESI);
|
||||
MOVobd(ECX, ESI, EDX);
|
||||
break;
|
||||
|
||||
@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
* be optimized if the previous instruction
|
||||
* was already of this type
|
||||
*/
|
||||
MOVid((uintptr_t)mem, ECX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVrd(EBP, ECX);
|
||||
MOVid(((int)ins->k - BPF_MEMWORDS) *
|
||||
sizeof(uint32_t), ESI);
|
||||
MOVomd(EAX, ECX, ESI);
|
||||
break;
|
||||
|
||||
case BPF_STX:
|
||||
MOVid((uintptr_t)mem, ECX);
|
||||
MOVid(ins->k * 4, ESI);
|
||||
MOVrd(EBP, ECX);
|
||||
MOVid(((int)ins->k - BPF_MEMWORDS) *
|
||||
sizeof(uint32_t), ESI);
|
||||
MOVomd(EDX, ECX, ESI);
|
||||
break;
|
||||
|
||||
@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
|
||||
}
|
||||
|
||||
#ifdef _KERNEL
|
||||
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
|
||||
M_NOWAIT, 0, ~0UL, 16, 0);
|
||||
stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
|
||||
if (stream.ibuf == NULL)
|
||||
break;
|
||||
#else
|
||||
stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
|
||||
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
|
||||
stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
|
||||
MAP_ANON, -1, 0);
|
||||
if (stream.ibuf == MAP_FAILED) {
|
||||
stream.ibuf = NULL;
|
||||
break;
|
||||
|
@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
|
||||
emitm(&stream, i32, 4); \
|
||||
} while (0)
|
||||
|
||||
/* subl i8,r32 */
|
||||
#define SUBib(i8, r32) do { \
|
||||
emitm(&stream, 0x83, 1); \
|
||||
emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \
|
||||
emitm(&stream, i8, 1); \
|
||||
} while (0)
|
||||
|
||||
/* mull r32 */
|
||||
#define MULrd(r32) do { \
|
||||
emitm(&stream, 0xf7, 1); \
|
||||
|
@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
|
||||
#include <net/bpf.h>
|
||||
#include <net/bpf_jitter.h>
|
||||
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
|
||||
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
|
||||
|
||||
static u_int bpf_jit_accept_all(u_char *, u_int, u_int);
|
||||
|
||||
@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
|
||||
|
||||
/* Allocate the filter structure */
|
||||
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
|
||||
M_BPFJIT, M_NOWAIT | M_ZERO);
|
||||
M_BPFJIT, M_NOWAIT);
|
||||
if (filter == NULL)
|
||||
return (NULL);
|
||||
|
||||
@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
|
||||
}
|
||||
|
||||
/* Create the binary */
|
||||
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
|
||||
filter->mem)) == NULL) {
|
||||
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
|
||||
free(filter, M_BPFJIT);
|
||||
return (NULL);
|
||||
}
|
||||
@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter)
|
||||
{
|
||||
|
||||
if (filter->func != bpf_jit_accept_all)
|
||||
contigfree(filter->func, filter->size, M_BPFJIT);
|
||||
free(filter->func, M_BPFJIT);
|
||||
free(filter, M_BPFJIT);
|
||||
}
|
||||
#else
|
||||
@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins)
|
||||
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
|
||||
if (filter == NULL)
|
||||
return (NULL);
|
||||
memset(filter, 0, sizeof(*filter));
|
||||
|
||||
/* No filter means accept all */
|
||||
if (fp == NULL || nins == 0) {
|
||||
@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
|
||||
}
|
||||
|
||||
/* Create the binary */
|
||||
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
|
||||
filter->mem)) == NULL) {
|
||||
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
|
||||
free(filter);
|
||||
return (NULL);
|
||||
}
|
||||
|
@ -54,7 +54,6 @@ typedef struct bpf_jit_filter {
|
||||
/* The native filtering binary, in the form of a bpf_filter_func. */
|
||||
bpf_filter_func func;
|
||||
size_t size;
|
||||
int mem[BPF_MEMWORDS]; /* Scratch memory */
|
||||
} bpf_jit_filter;
|
||||
|
||||
/*
|
||||
|
Loading…
Reference in New Issue
Block a user