d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Allocate scratch memory on stack instead of pre-allocating it with

Browse Source 
the filter as we do from bpf_filter()[1].
- Revert experimental use of contigmalloc(9)/contigfree(9).  It has no
performance benefit over malloc(9)/free(9)[2].

Requested by:	rwatson[1]
Pointed out by:	rwatson, jhb, alc[2]
This commit is contained in:
jkim 2009-11-20 18:49:20 +00:00
parent ca5edf0fbf
commit 052bc52af7
6 changed files with 98 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
sys/amd64/amd64/bpf_jit_machdep.c
View File

@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <amd64/amd64/bpf_jit_machdep.h> #include <amd64/amd64/bpf_jit_machdep.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/* /*
* emit routine to update the jump table * emit routine to update the jump table
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
* Function that does the real stuff * Function that does the real stuff
*/ */
bpf_filter_func bpf_filter_func
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{ {
bpf_bin_stream stream; bpf_bin_stream stream;
struct bpf_insn *ins; struct bpf_insn *ins;
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* Allocate the reference table for the jumps */ /* Allocate the reference table for the jumps */
#ifdef _KERNEL #ifdef _KERNEL
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
M_BPFJIT, M_NOWAIT);
#else #else
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif #endif
if (stream.refs == NULL) if (stream.refs == NULL)
return (NULL); return (NULL);
@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
ins = prog; ins = prog;
/* create the procedure header */ /* create the procedure header */
MOVrq2(RBX, R8); PUSH(RBP);
MOVrq(RDI, RBX); MOVrq(RSP, RBP);
SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP);
MOVrq2(RDI, R8);
MOVrd2(ESI, R9D); MOVrd2(ESI, R9D);
MOVrd(EDX, EDI); MOVrd(EDX, EDI);
@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
case BPF_RET|BPF_K: case BPF_RET|BPF_K:
MOVid(ins->k, EAX); MOVid(ins->k, EAX);
MOVrq3(R8, RBX); LEAVE_RET();
RET();
break; break;
case BPF_RET|BPF_A: case BPF_RET|BPF_A:
MOVrq3(R8, RBX); LEAVE_RET();
RET();
break; break;
case BPF_LD|BPF_W|BPF_ABS: case BPF_LD|BPF_W|BPF_ABS:
@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX); MOVrd(EDI, ECX);
SUBrd(ESI, ECX); SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX); CMPid(sizeof(int32_t), ECX);
JAEb(6); JAEb(4);
ZEROrd(EAX); ZEROrd(EAX);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
MOVobd(RBX, RSI, EAX); MOVobd(RCX, RSI, EAX);
BSWAP(EAX); BSWAP(EAX);
break; break;
@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX); MOVrd(EDI, ECX);
SUBrd(ESI, ECX); SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX); CMPid(sizeof(int16_t), ECX);
JAEb(4); JAEb(2);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
MOVobw(RBX, RSI, AX); MOVobw(RCX, RSI, AX);
SWAP_AX(); SWAP_AX();
break; break;
@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
ZEROrd(EAX); ZEROrd(EAX);
MOVid(ins->k, ESI); MOVid(ins->k, ESI);
CMPrd(EDI, ESI); CMPrd(EDI, ESI);
JBb(4); JBb(2);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
MOVobb(RBX, RSI, AL); MOVobb(RCX, RSI, AL);
break; break;
case BPF_LD|BPF_W|BPF_LEN: case BPF_LD|BPF_W|BPF_LEN:
@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX); MOVrd(EDI, ECX);
SUBrd(ESI, ECX); SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX); CMPid(sizeof(int32_t), ECX);
JAEb(6); JAEb(4);
ZEROrd(EAX); ZEROrd(EAX);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
MOVobd(RBX, RSI, EAX); MOVobd(RCX, RSI, EAX);
BSWAP(EAX); BSWAP(EAX);
break; break;
@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX); MOVrd(EDI, ECX);
SUBrd(ESI, ECX); SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX); CMPid(sizeof(int16_t), ECX);
JAEb(4); JAEb(2);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
MOVobw(RBX, RSI, AX); MOVobw(RCX, RSI, AX);
SWAP_AX(); SWAP_AX();
break; break;
@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
MOVrd(EDI, ECX); MOVrd(EDI, ECX);
SUBrd(EDX, ECX); SUBrd(EDX, ECX);
CMPrd(ESI, ECX); CMPrd(ESI, ECX);
JAb(4); JAb(2);
MOVrq3(R8, RBX); LEAVE_RET();
RET(); MOVrq3(R8, RCX);
ADDrd(EDX, ESI); ADDrd(EDX, ESI);
MOVobb(RBX, RSI, AL); MOVobb(RCX, RSI, AL);
break; break;
case BPF_LDX|BPF_MSH|BPF_B: case BPF_LDX|BPF_MSH|BPF_B:
MOVid(ins->k, ESI); MOVid(ins->k, ESI);
CMPrd(EDI, ESI); CMPrd(EDI, ESI);
JBb(6); JBb(4);
ZEROrd(EAX); ZEROrd(EAX);
MOVrq3(R8, RBX); LEAVE_RET();
RET();
ZEROrd(EDX); ZEROrd(EDX);
MOVobb(RBX, RSI, DL); MOVrq3(R8, RCX);
MOVobb(RCX, RSI, DL);
ANDib(0x0f, DL); ANDib(0x0f, DL);
SHLib(2, EDX); SHLib(2, EDX);
break; break;
@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
break; break;
case BPF_LD|BPF_MEM: case BPF_LD|BPF_MEM:
MOViq((uintptr_t)mem, RCX); MOVid(ins->k * sizeof(uint32_t), ESI);
MOVid(ins->k * 4, ESI); MOVobd(RSP, RSI, EAX);
MOVobd(RCX, RSI, EAX);
break; break;
case BPF_LDX|BPF_MEM: case BPF_LDX|BPF_MEM:
MOViq((uintptr_t)mem, RCX); MOVid(ins->k * sizeof(uint32_t), ESI);
MOVid(ins->k * 4, ESI); MOVobd(RSP, RSI, EDX);
MOVobd(RCX, RSI, EDX);
break; break;
case BPF_ST: case BPF_ST:
@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
* be optimized if the previous instruction * be optimized if the previous instruction
* was already of this type * was already of this type
*/ */
MOViq((uintptr_t)mem, RCX); MOVid(ins->k * sizeof(uint32_t), ESI);
MOVid(ins->k * 4, ESI); MOVomd(EAX, RSP, RSI);
MOVomd(EAX, RCX, RSI);
break; break;
case BPF_STX: case BPF_STX:
MOViq((uintptr_t)mem, RCX); MOVid(ins->k * sizeof(uint32_t), ESI);
MOVid(ins->k * 4, ESI); MOVomd(EDX, RSP, RSI);
MOVomd(EDX, RCX, RSI);
break; break;
case BPF_JMP|BPF_JA: case BPF_JMP|BPF_JA:
@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
case BPF_ALU|BPF_DIV|BPF_X: case BPF_ALU|BPF_DIV|BPF_X:
TESTrd(EDX, EDX); TESTrd(EDX, EDX);
JNEb(6); JNEb(4);
ZEROrd(EAX); ZEROrd(EAX);
MOVrq3(R8, RBX); LEAVE_RET();
RET();
MOVrd(EDX, ECX); MOVrd(EDX, ECX);
ZEROrd(EDX); ZEROrd(EDX);
DIVrd(ECX); DIVrd(ECX);
@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
} }
#ifdef _KERNEL #ifdef _KERNEL
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
M_NOWAIT, 0, ~0UL, 16, 0);
if (stream.ibuf == NULL) if (stream.ibuf == NULL)
break; break;
#else #else
stream.ibuf = (char *)mmap(NULL, stream.cur_ip, stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) { if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL; stream.ibuf = NULL;
break; break;

18
sys/amd64/amd64/bpf_jit_machdep.h
View File

@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, 0xc486, 2); \ emitm(&stream, 0xc486, 2); \
} while (0) } while (0)
/* ret */ /* pushq r64 */
#define RET() do { \ #define PUSH(r64) do { \
emitm(&stream, 0xc3, 1); \ emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \
} while (0)
/* leave/ret */
#define LEAVE_RET() do { \
emitm(&stream, 0xc3c9, 2); \
} while (0) } while (0)
/* addl sr32,dr32 */ /* addl sr32,dr32 */
@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, i32, 4); \ emitm(&stream, i32, 4); \
} while (0) } while (0)
/* subq i8,r64 */
#define SUBib(i8, r64) do { \
emitm(&stream, 0x8348, 2); \
emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \
emitm(&stream, i8, 1); \
} while (0)
/* mull r32 */ /* mull r32 */
#define MULrd(r32) do { \ #define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \ emitm(&stream, 0xf7, 1); \

37
sys/i386/i386/bpf_jit_machdep.c
View File

@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$");
#include <i386/i386/bpf_jit_machdep.h> #include <i386/i386/bpf_jit_machdep.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
/* /*
* emit routine to update the jump table * emit routine to update the jump table
@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len)
* Function that does the real stuff * Function that does the real stuff
*/ */
bpf_filter_func bpf_filter_func
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{ {
bpf_bin_stream stream; bpf_bin_stream stream;
struct bpf_insn *ins; struct bpf_insn *ins;
@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* Allocate the reference table for the jumps */ /* Allocate the reference table for the jumps */
#ifdef _KERNEL #ifdef _KERNEL
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT);
M_BPFJIT, M_NOWAIT);
#else #else
stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); stream.refs = malloc((nins + 1) * sizeof(u_int));
#endif #endif
if (stream.refs == NULL) if (stream.refs == NULL)
return (NULL); return (NULL);
@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
/* create the procedure header */ /* create the procedure header */
PUSH(EBP); PUSH(EBP);
MOVrd(ESP, EBP); MOVrd(ESP, EBP);
SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
PUSH(EDI); PUSH(EDI);
PUSH(ESI); PUSH(ESI);
PUSH(EBX); PUSH(EBX);
@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
break; break;
case BPF_LD|BPF_MEM: case BPF_LD|BPF_MEM:
MOVid((uintptr_t)mem, ECX); MOVrd(EBP, ECX);
MOVid(ins->k * 4, ESI); MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EAX); MOVobd(ECX, ESI, EAX);
break; break;
case BPF_LDX|BPF_MEM: case BPF_LDX|BPF_MEM:
MOVid((uintptr_t)mem, ECX); MOVrd(EBP, ECX);
MOVid(ins->k * 4, ESI); MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EDX); MOVobd(ECX, ESI, EDX);
break; break;
@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
* be optimized if the previous instruction * be optimized if the previous instruction
* was already of this type * was already of this type
*/ */
MOVid((uintptr_t)mem, ECX); MOVrd(EBP, ECX);
MOVid(ins->k * 4, ESI); MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EAX, ECX, ESI); MOVomd(EAX, ECX, ESI);
break; break;
case BPF_STX: case BPF_STX:
MOVid((uintptr_t)mem, ECX); MOVrd(EBP, ECX);
MOVid(ins->k * 4, ESI); MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EDX, ECX, ESI); MOVomd(EDX, ECX, ESI);
break; break;
@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem)
} }
#ifdef _KERNEL #ifdef _KERNEL
stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT);
M_NOWAIT, 0, ~0UL, 16, 0);
if (stream.ibuf == NULL) if (stream.ibuf == NULL)
break; break;
#else #else
stream.ibuf = (char *)mmap(NULL, stream.cur_ip, stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE,
PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) { if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL; stream.ibuf = NULL;
break; break;

7
sys/i386/i386/bpf_jit_machdep.h
View File

@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n);
emitm(&stream, i32, 4); \ emitm(&stream, i32, 4); \
} while (0) } while (0)
/* subl i8,r32 */
#define SUBib(i8, r32) do { \
emitm(&stream, 0x83, 1); \
emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \
emitm(&stream, i8, 1); \
} while (0)
/* mull r32 */ /* mull r32 */
#define MULrd(r32) do { \ #define MULrd(r32) do { \
emitm(&stream, 0xf7, 1); \ emitm(&stream, 0xf7, 1); \

13
sys/net/bpf_jitter.c
View File

@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$");
#include <net/bpf.h> #include <net/bpf.h>
#include <net/bpf_jitter.h> #include <net/bpf_jitter.h>
bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *);
static u_int bpf_jit_accept_all(u_char *, u_int, u_int); static u_int bpf_jit_accept_all(u_char *, u_int, u_int);
@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
/* Allocate the filter structure */ /* Allocate the filter structure */
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter), filter = (struct bpf_jit_filter *)malloc(sizeof(*filter),
M_BPFJIT, M_NOWAIT | M_ZERO); M_BPFJIT, M_NOWAIT);
if (filter == NULL) if (filter == NULL)
return (NULL); return (NULL);
@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
} }
/* Create the binary */ /* Create the binary */
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
filter->mem)) == NULL) {
free(filter, M_BPFJIT); free(filter, M_BPFJIT);
return (NULL); return (NULL);
} }
@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter)
{ {
if (filter->func != bpf_jit_accept_all) if (filter->func != bpf_jit_accept_all)
contigfree(filter->func, filter->size, M_BPFJIT); free(filter->func, M_BPFJIT);
free(filter, M_BPFJIT); free(filter, M_BPFJIT);
} }
#else #else
@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins)
filter = (struct bpf_jit_filter *)malloc(sizeof(*filter)); filter = (struct bpf_jit_filter *)malloc(sizeof(*filter));
if (filter == NULL) if (filter == NULL)
return (NULL); return (NULL);
memset(filter, 0, sizeof(*filter));
/* No filter means accept all */ /* No filter means accept all */
if (fp == NULL || nins == 0) { if (fp == NULL || nins == 0) {
@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins)
} }
/* Create the binary */ /* Create the binary */
if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) {
filter->mem)) == NULL) {
free(filter); free(filter);
return (NULL); return (NULL);
} }

1
sys/net/bpf_jitter.h
View File

@ -54,7 +54,6 @@ typedef struct bpf_jit_filter {
/* The native filtering binary, in the form of a bpf_filter_func. */ /* The native filtering binary, in the form of a bpf_filter_func. */
bpf_filter_func func; bpf_filter_func func;
size_t size; size_t size;
int mem[BPF_MEMWORDS]; /* Scratch memory */
} bpf_jit_filter; } bpf_jit_filter;
/* /*