diff --git a/sys/amd64/amd64/bpf_jit_machdep.c b/sys/amd64/amd64/bpf_jit_machdep.c index da2b0f1712fe..bbaebed29fe7 100644 --- a/sys/amd64/amd64/bpf_jit_machdep.c +++ b/sys/amd64/amd64/bpf_jit_machdep.c @@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$"); #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); /* * emit routine to update the jump table @@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len) * Function that does the real stuff */ bpf_filter_func -bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) +bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; @@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) /* Allocate the reference table for the jumps */ #ifdef _KERNEL - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), - M_BPFJIT, M_NOWAIT); + stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT); #else - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); + stream.refs = malloc((nins + 1) * sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); @@ -137,8 +136,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) ins = prog; /* create the procedure header */ - MOVrq2(RBX, R8); - MOVrq(RDI, RBX); + PUSH(RBP); + MOVrq(RSP, RBP); + SUBib(BPF_MEMWORDS * sizeof(uint32_t), RSP); + MOVrq2(RDI, R8); MOVrd2(ESI, R9D); MOVrd(EDX, EDI); @@ -155,13 +156,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) case BPF_RET|BPF_K: MOVid(ins->k, EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); break; case BPF_RET|BPF_A: - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); break; case BPF_LD|BPF_W|BPF_ABS: @@ -171,11 +170,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); - JAEb(6); + JAEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); - MOVobd(RBX, RSI, EAX); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; @@ -187,10 +186,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); - JAEb(4); - MOVrq3(R8, RBX); - RET(); - MOVobw(RBX, RSI, AX); + JAEb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobw(RCX, RSI, AX); SWAP_AX(); break; @@ -198,10 +197,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) ZEROrd(EAX); MOVid(ins->k, ESI); CMPrd(EDI, ESI); - JBb(4); - MOVrq3(R8, RBX); - RET(); - MOVobb(RBX, RSI, AL); + JBb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobb(RCX, RSI, AL); break; case BPF_LD|BPF_W|BPF_LEN: @@ -224,11 +223,11 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int32_t), ECX); - JAEb(6); + JAEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); - MOVobd(RBX, RSI, EAX); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobd(RCX, RSI, EAX); BSWAP(EAX); break; @@ -245,10 +244,10 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) MOVrd(EDI, ECX); SUBrd(ESI, ECX); CMPid(sizeof(int16_t), ECX); - JAEb(4); - MOVrq3(R8, RBX); - RET(); - MOVobw(RBX, RSI, AX); + JAEb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); + MOVobw(RCX, RSI, AX); SWAP_AX(); break; @@ -260,22 +259,22 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) MOVrd(EDI, ECX); SUBrd(EDX, ECX); CMPrd(ESI, ECX); - JAb(4); - MOVrq3(R8, RBX); - RET(); + JAb(2); + LEAVE_RET(); + MOVrq3(R8, RCX); ADDrd(EDX, ESI); - MOVobb(RBX, RSI, AL); + MOVobb(RCX, RSI, AL); break; case BPF_LDX|BPF_MSH|BPF_B: MOVid(ins->k, ESI); CMPrd(EDI, ESI); - JBb(6); + JBb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); ZEROrd(EDX); - MOVobb(RBX, RSI, DL); + MOVrq3(R8, RCX); + MOVobb(RCX, RSI, DL); ANDib(0x0f, DL); SHLib(2, EDX); break; @@ -289,15 +288,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) break; case BPF_LD|BPF_MEM: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVobd(RCX, RSI, EAX); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVobd(RSP, RSI, EAX); break; case BPF_LDX|BPF_MEM: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVobd(RCX, RSI, EDX); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVobd(RSP, RSI, EDX); break; case BPF_ST: @@ -306,15 +303,13 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) * be optimized if the previous instruction * was already of this type */ - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVomd(EAX, RCX, RSI); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVomd(EAX, RSP, RSI); break; case BPF_STX: - MOViq((uintptr_t)mem, RCX); - MOVid(ins->k * 4, ESI); - MOVomd(EDX, RCX, RSI); + MOVid(ins->k * sizeof(uint32_t), ESI); + MOVomd(EDX, RSP, RSI); break; case BPF_JMP|BPF_JA: @@ -394,10 +389,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) case BPF_ALU|BPF_DIV|BPF_X: TESTrd(EDX, EDX); - JNEb(6); + JNEb(4); ZEROrd(EAX); - MOVrq3(R8, RBX); - RET(); + LEAVE_RET(); MOVrd(EDX, ECX); ZEROrd(EDX); DIVrd(ECX); @@ -490,13 +484,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) } #ifdef _KERNEL - stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, - M_NOWAIT, 0, ~0UL, 16, 0); + stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else - stream.ibuf = (char *)mmap(NULL, stream.cur_ip, - PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); + stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE, + MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; diff --git a/sys/amd64/amd64/bpf_jit_machdep.h b/sys/amd64/amd64/bpf_jit_machdep.h index 154bbaecd4ff..463418763a0b 100644 --- a/sys/amd64/amd64/bpf_jit_machdep.h +++ b/sys/amd64/amd64/bpf_jit_machdep.h @@ -215,9 +215,14 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, 0xc486, 2); \ } while (0) -/* ret */ -#define RET() do { \ - emitm(&stream, 0xc3, 1); \ +/* pushq r64 */ +#define PUSH(r64) do { \ + emitm(&stream, (5 << 4) | (0 << 3) | (r64 & 0x7), 1); \ +} while (0) + +/* leave/ret */ +#define LEAVE_RET() do { \ + emitm(&stream, 0xc3c9, 2); \ } while (0) /* addl sr32,dr32 */ @@ -253,6 +258,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, i32, 4); \ } while (0) +/* subq i8,r64 */ +#define SUBib(i8, r64) do { \ + emitm(&stream, 0x8348, 2); \ + emitm(&stream, (29 << 3) | (r64 & 0x7), 1); \ + emitm(&stream, i8, 1); \ +} while (0) + /* mull r32 */ #define MULrd(r32) do { \ emitm(&stream, 0xf7, 1); \ diff --git a/sys/i386/i386/bpf_jit_machdep.c b/sys/i386/i386/bpf_jit_machdep.c index 2b40b19b2eb2..8e51f7bd0cbe 100644 --- a/sys/i386/i386/bpf_jit_machdep.c +++ b/sys/i386/i386/bpf_jit_machdep.c @@ -53,7 +53,7 @@ __FBSDID("$FreeBSD$"); #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); /* * emit routine to update the jump table @@ -97,7 +97,7 @@ emit_code(bpf_bin_stream *stream, u_int value, u_int len) * Function that does the real stuff */ bpf_filter_func -bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) +bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size) { bpf_bin_stream stream; struct bpf_insn *ins; @@ -111,10 +111,9 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) /* Allocate the reference table for the jumps */ #ifdef _KERNEL - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int), - M_BPFJIT, M_NOWAIT); + stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT, M_NOWAIT); #else - stream.refs = (u_int *)malloc((nins + 1) * sizeof(u_int)); + stream.refs = malloc((nins + 1) * sizeof(u_int)); #endif if (stream.refs == NULL) return (NULL); @@ -139,6 +138,7 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) /* create the procedure header */ PUSH(EBP); MOVrd(ESP, EBP); + SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP); PUSH(EDI); PUSH(ESI); PUSH(EBX); @@ -310,14 +310,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) break; case BPF_LD|BPF_MEM: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EAX); break; case BPF_LDX|BPF_MEM: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVobd(ECX, ESI, EDX); break; @@ -327,14 +329,16 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) * be optimized if the previous instruction * was already of this type */ - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVomd(EAX, ECX, ESI); break; case BPF_STX: - MOVid((uintptr_t)mem, ECX); - MOVid(ins->k * 4, ESI); + MOVrd(EBP, ECX); + MOVid(((int)ins->k - BPF_MEMWORDS) * + sizeof(uint32_t), ESI); MOVomd(EDX, ECX, ESI); break; @@ -513,13 +517,12 @@ bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size, int *mem) } #ifdef _KERNEL - stream.ibuf = (char *)contigmalloc(stream.cur_ip, M_BPFJIT, - M_NOWAIT, 0, ~0UL, 16, 0); + stream.ibuf = malloc(stream.cur_ip, M_BPFJIT, M_NOWAIT); if (stream.ibuf == NULL) break; #else - stream.ibuf = (char *)mmap(NULL, stream.cur_ip, - PROT_READ | PROT_WRITE, MAP_ANON, -1, 0); + stream.ibuf = mmap(NULL, stream.cur_ip, PROT_READ | PROT_WRITE, + MAP_ANON, -1, 0); if (stream.ibuf == MAP_FAILED) { stream.ibuf = NULL; break; diff --git a/sys/i386/i386/bpf_jit_machdep.h b/sys/i386/i386/bpf_jit_machdep.h index 0ec94d298b8b..7e6f2b0b2fc7 100644 --- a/sys/i386/i386/bpf_jit_machdep.h +++ b/sys/i386/i386/bpf_jit_machdep.h @@ -203,6 +203,13 @@ typedef void (*emit_func)(bpf_bin_stream *stream, u_int value, u_int n); emitm(&stream, i32, 4); \ } while (0) +/* subl i8,r32 */ +#define SUBib(i8, r32) do { \ + emitm(&stream, 0x83, 1); \ + emitm(&stream, (29 << 3) | (r32 & 0x7), 1); \ + emitm(&stream, i8, 1); \ +} while (0) + /* mull r32 */ #define MULrd(r32) do { \ emitm(&stream, 0xf7, 1); \ diff --git a/sys/net/bpf_jitter.c b/sys/net/bpf_jitter.c index 112b8734839b..a72d2d4c6b12 100644 --- a/sys/net/bpf_jitter.c +++ b/sys/net/bpf_jitter.c @@ -51,7 +51,7 @@ __FBSDID("$FreeBSD$"); #include #include -bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *, int *); +bpf_filter_func bpf_jit_compile(struct bpf_insn *, u_int, size_t *); static u_int bpf_jit_accept_all(u_char *, u_int, u_int); @@ -70,7 +70,7 @@ bpf_jitter(struct bpf_insn *fp, int nins) /* Allocate the filter structure */ filter = (struct bpf_jit_filter *)malloc(sizeof(*filter), - M_BPFJIT, M_NOWAIT | M_ZERO); + M_BPFJIT, M_NOWAIT); if (filter == NULL) return (NULL); @@ -81,8 +81,7 @@ bpf_jitter(struct bpf_insn *fp, int nins) } /* Create the binary */ - if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, - filter->mem)) == NULL) { + if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) { free(filter, M_BPFJIT); return (NULL); } @@ -95,7 +94,7 @@ bpf_destroy_jit_filter(bpf_jit_filter *filter) { if (filter->func != bpf_jit_accept_all) - contigfree(filter->func, filter->size, M_BPFJIT); + free(filter->func, M_BPFJIT); free(filter, M_BPFJIT); } #else @@ -108,7 +107,6 @@ bpf_jitter(struct bpf_insn *fp, int nins) filter = (struct bpf_jit_filter *)malloc(sizeof(*filter)); if (filter == NULL) return (NULL); - memset(filter, 0, sizeof(*filter)); /* No filter means accept all */ if (fp == NULL || nins == 0) { @@ -117,8 +115,7 @@ bpf_jitter(struct bpf_insn *fp, int nins) } /* Create the binary */ - if ((filter->func = bpf_jit_compile(fp, nins, &filter->size, - filter->mem)) == NULL) { + if ((filter->func = bpf_jit_compile(fp, nins, &filter->size)) == NULL) { free(filter); return (NULL); } diff --git a/sys/net/bpf_jitter.h b/sys/net/bpf_jitter.h index fa998687b93a..90a1ff5fbb07 100644 --- a/sys/net/bpf_jitter.h +++ b/sys/net/bpf_jitter.h @@ -54,7 +54,6 @@ typedef struct bpf_jit_filter { /* The native filtering binary, in the form of a bpf_filter_func. */ bpf_filter_func func; size_t size; - int mem[BPF_MEMWORDS]; /* Scratch memory */ } bpf_jit_filter; /*