d/freebsd-dev
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Allocate scratch memory on stack instead of pre-allocating it with

Browse Source 
the filter as we do from bpf_filter()[1].
- Revert experimental use of contigmalloc(9)/contigfree(9).  It has no
performance benefit over malloc(9)/free(9)[2].

Requested by:	rwatson[1]
Pointed out by:	rwatson, jhb, alc[2]
This commit is contained in:
Jung-uk Kim 2009-11-20 18:49:20 +00:00
parent 0c8e3b18f0
commit 5ecf77367c
Notes: svn2git 2020-12-20 02:59:44 +00:00 
svn path=/head/; revision=199603
6 changed files with 98 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
sys/amd64/amd64/bpf_jit_machdep.c
View File

@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <amd64/amd64/bpf_jit_machdep.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/*
* emit routine to update the jump table
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
* Function that does the real stuff
*/
bpf_filter_func
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{
bpf_bin_stream stream;
struct bpf_insn *ins;
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* Allocate the reference table for the jumps */
#ifdef _KERNEL
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
M_BPFJIT, M_NOWAIT);
stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
#else
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif
if (stream.refs == NULL)
return (NULL);
@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
ins = prog;
/* create the procedure header */
MOVrq2(RBX, R8);
MOVrq(RDI, RBX);
PUSH(RBP);
MOVrq(RSP, RBP);
SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
MOVrq2(RDI, R8);
MOVrd2(ESI, R9D);
MOVrd(EDX, EDI);
@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
case BPF_RET|BPF_K:
MOVid(ins->k, EAX);
MOVrq3(R8, RBX);
RET();
LEAVE_RET();
break;
case BPF_RET|BPF_A:
MOVrq3(R8, RBX);
RET();
LEAVE_RET();
break;
case BPF_LD|BPF_W|BPF_ABS:
@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
JAEb(6);
JAEb(4);
ZEROrd(EAX);
MOVrq3(R8, RBX);
RET();
MOVobd(RBX, RSI, EAX);
LEAVE_RET();
MOVrq3(R8, RCX);
MOVobd(RCX, RSI, EAX);
BSWAP(EAX);
break;
@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
JAEb(4);
MOVrq3(R8, RBX);
RET();
MOVobw(RBX, RSI, AX);
JAEb(2);
LEAVE_RET();
MOVrq3(R8, RCX);
MOVobw(RCX, RSI, AX);
SWAP_AX();
break;
@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
ZEROrd(EAX);
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JBb(4);
MOVrq3(R8, RBX);
RET();
MOVobb(RBX, RSI, AL);
JBb(2);
LEAVE_RET();
MOVrq3(R8, RCX);
MOVobb(RCX, RSI, AL);
break;
case BPF_LD|BPF_W|BPF_LEN:
@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
JAEb(6);
JAEb(4);
ZEROrd(EAX);
MOVrq3(R8, RBX);
RET();
MOVobd(RBX, RSI, EAX);
LEAVE_RET();
MOVrq3(R8, RCX);
MOVobd(RCX, RSI, EAX);
BSWAP(EAX);
break;
@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
JAEb(4);
MOVrq3(R8, RBX);
RET();
MOVobw(RBX, RSI, AX);
JAEb(2);
LEAVE_RET();
MOVrq3(R8, RCX);
MOVobw(RCX, RSI, AX);
SWAP_AX();
break;
@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX);
SUBrd(EDX, ECX);
CMPrd(ESI, ECX);
JAb(4);
MOVrq3(R8, RBX);
RET();
JAb(2);
LEAVE_RET();
MOVrq3(R8, RCX);
ADDrd(EDX, ESI);
MOVobb(RBX, RSI, AL);
MOVobb(RCX, RSI, AL);
break;
case BPF_LDX|BPF_MSH|BPF_B:
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JBb(6);
JBb(4);
ZEROrd(EAX);
MOVrq3(R8, RBX);
RET();
LEAVE_RET();
ZEROrd(EDX);
MOVobb(RBX, RSI, DL);
MOVrq3(R8, RCX);
MOVobb(RCX, RSI, DL);
ANDib(0x0f, DL);
SHLib(2, EDX);
break;
@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
break;
case BPF_LD|BPF_MEM:
MOViq((uintptr_t)mem, RCX);
MOVid(ins->k * 4, ESI);
MOVobd(RCX, RSI, EAX);
MOVid(ins->k * sizeof(uint32_t), ESI);
MOVobd(RSP, RSI, EAX);
break;
case BPF_LDX|BPF_MEM:
MOViq((uintptr_t)mem, RCX);
MOVid(ins->k * 4, ESI);
MOVobd(RCX, RSI, EDX);
MOVid(ins->k * sizeof(uint32_t), ESI);
MOVobd(RSP, RSI, EDX);
break;
case BPF_ST:
@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
* be optimized if the previous instruction
* was already of this type
*/
MOViq((uintptr_t)mem, RCX);
MOVid(ins->k * 4, ESI);
MOVomd(EAX, RCX, RSI);
MOVid(ins->k * sizeof(uint32_t), ESI);
MOVomd(EAX, RSP, RSI);
break;
case BPF_STX:
MOViq((uintptr_t)mem, RCX);
MOVid(ins->k * 4, ESI);
MOVomd(EDX, RCX, RSI);
MOVid(ins->k * sizeof(uint32_t), ESI);
MOVomd(EDX, RSP, RSI);
break;
case BPF_JMP|BPF_JA:
@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
case BPF_ALU|BPF_DIV|BPF_X:
TESTrd(EDX, EDX);
JNEb(6);
JNEb(4);
ZEROrd(EAX);
MOVrq3(R8, RBX);
RET();
LEAVE_RET();
MOVrd(EDX, ECX);
ZEROrd(EDX);
DIVrd(ECX);
@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
}
#ifdef _KERNEL
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
M_NOWAIT, 0, ~0UL, 16, 0);
stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
if (stream.ibuf == NULL)
break;
#else
stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL;
break;

18
sys/amd64/amd64/bpf_jit_machdep.h
View File

@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, 0xc486, 2); \
} while (0)
/* ret */
#define RET() do { \
emitm(&stream, 0xc3, 1); \
/* pushq r64 */
#define PUSH(r64) do { \
emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \
} while (0)
/* leave/ret */
#define LEAVE_RET() do { \
emitm(&stream, 0xc3c9, 2); \
} while (0)
/* addl sr32,dr32 */
@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, i32, 4); \
} while (0)
/* subq i8,r64 */
#define SUBib(i8, r64) do { \
emitm(&stream, 0x8348, 2); \
emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \
emitm(&stream, i8, 1); \
} while (0)
/* mull r32 */
#define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \

37
sys/i386/i386/bpf_jit_machdep.c
View File

@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <i386/i386/bpf_jit_machdep.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/*
* emit routine to update the jump table
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
* Function that does the real stuff
*/
bpf_filter_func
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{
bpf_bin_stream stream;
struct bpf_insn *ins;
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* Allocate the reference table for the jumps */
#ifdef _KERNEL
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int),
M_BPFJIT, M_NOWAIT);
stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
#else
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int));
stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif
if (stream.refs == NULL)
return (NULL);
@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* create the procedure header */
PUSH(EBP);
MOVrd(ESP, EBP);
SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
PUSH(EDI);
PUSH(ESI);
PUSH(EBX);
@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
break;
case BPF_LD|BPF_MEM:
MOVid((uintptr_t)mem, ECX);
MOVid(ins->k * 4, ESI);
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EAX);
break;
case BPF_LDX|BPF_MEM:
MOVid((uintptr_t)mem, ECX);
MOVid(ins->k * 4, ESI);
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EDX);
break;
@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
* be optimized if the previous instruction
* was already of this type
*/
MOVid((uintptr_t)mem, ECX);
MOVid(ins->k * 4, ESI);
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EAX, ECX, ESI);
break;
case BPF_STX:
MOVid((uintptr_t)mem, ECX);
MOVid(ins->k * 4, ESI);
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EDX, ECX, ESI);
break;
@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
}
#ifdef _KERNEL
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT,
M_NOWAIT, 0, ~0UL, 16, 0);
stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
if (stream.ibuf == NULL)
break;
#else
stream.ibuf = (char *)mmap(NULL, stream.cur_ip,
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0);
stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL;
break;

7
sys/i386/i386/bpf_jit_machdep.h
View File

@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, i32, 4); \
} while (0)
/* subl i8,r32 */
#define SUBib(i8, r32) do { \
emitm(&stream, 0x83, 1); \
emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \
emitm(&stream, i8, 1); \
} while (0)
/* mull r32 */
#define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \

13
sys/net/bpf_jitter.c
View File

@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include <net/bpf.h>
#include <net/bpf_jitter.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *);
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
static u_int bpf_jit_accept_all(u_char *, u_int, u_int);
@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
/* Allocate the filter structure */
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
M_BPFJIT, M_NOWAIT | M_ZERO);
M_BPFJIT, M_NOWAIT);
if (filter == NULL)
return (NULL);
@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
}
/* Create the binary */
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
filter->mem)) == NULL) {
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
free(filter, M_BPFJIT);
return (NULL);
}
@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter)
{
if (filter->func != bpf_jit_accept_all)
contigfree(filter->func, filter->size, M_BPFJIT);
free(filter->func, M_BPFJIT);
free(filter, M_BPFJIT);
}
#else
@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins)
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
if (filter == NULL)
return (NULL);
memset(filter, 0, sizeof(*filter));
/* No filter means accept all */
if (fp == NULL || nins == 0) {
@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
}
/* Create the binary */
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size,
filter->mem)) == NULL) {
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
free(filter);
return (NULL);
}

1
sys/net/bpf_jitter.h
View File

@ -54,7 +54,6 @@ typedef struct bpf_jit_filter {
/* The native filtering binary, in the form of a bpf_filter_func. */
bpf_filter_func func;
size_t size;
int mem[BPF_MEMWORDS]; /* Scratch memory */
} bpf_jit_filter;
/*