diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index 08948c0904e0..c49547d8fa66 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -463,8 +463,16 @@ fast_syscall_common: movq PCPU(SCRATCH_RSP),%r11 /* %r11 already saved */ movq %r11,TF_RSP(%rsp) /* user stack pointer */ movq PCPU(SCRATCH_RAX),%rax + /* + * Save a few arg registers early to free them for use in + * handle_ibrs_entry(). %r10 is especially tricky. It is not an + * arg register, but it holds the arg register %rcx. Profiling + * preserves %rcx, but may clobber %r10. Profiling may also + * clobber %r11, but %r11 (original %eflags) has been saved. + */ movq %rax,TF_RAX(%rsp) /* syscall number */ movq %rdx,TF_RDX(%rsp) /* arg 3 */ + movq %r10,TF_RCX(%rsp) /* arg 4 */ SAVE_SEGS call handle_ibrs_entry movq PCPU(CURPCB),%r11 @@ -475,7 +483,6 @@ fast_syscall_common: movq $2,TF_ERR(%rsp) movq %rdi,TF_RDI(%rsp) /* arg 1 */ movq %rsi,TF_RSI(%rsp) /* arg 2 */ - movq %r10,TF_RCX(%rsp) /* arg 4 */ movq %r8,TF_R8(%rsp) /* arg 5 */ movq %r9,TF_R9(%rsp) /* arg 6 */ movq %rbx,TF_RBX(%rsp) /* C preserved */ diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index a485f4a8b06c..2ee5032312a9 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -516,7 +516,9 @@ doreti_exit: 1: testl $SEL_RPL_MASK, TF_CS(%esp) jz doreti_popl_fs 2: movl $handle_ibrs_exit,%eax + pushl %ecx /* preserve enough call-used regs */ call *%eax + popl %ecx movl %esp, %esi movl PCPU(TRAMPSTK), %edx subl %ecx, %edx diff --git a/sys/i386/i386/prof_machdep.c b/sys/i386/i386/prof_machdep.c index 430e10cc5334..a6b4a9a24e56 100644 --- a/sys/i386/i386/prof_machdep.c +++ b/sys/i386/i386/prof_machdep.c @@ -117,6 +117,9 @@ __mcount: \n\ .mcount_exit: \n\ ret $0 \n\ "); + +void __mcount(void); +void (*__mcountp)(void) = __mcount; #else /* !__GNUCLIKE_ASM */ #error "this file needs to be ported to your compiler" #endif /* __GNUCLIKE_ASM */ @@ -163,6 +166,9 @@ GMON_PROF_HIRES = 4 \n\ "); #endif /* __GNUCLIKE_ASM */ +void __mexitcount(void); +void (*__mexitcountp)(void) = __mexitcount; + /* * Return the time elapsed since the last call. The units are machine- * dependent. diff --git a/sys/i386/i386/support.s b/sys/i386/i386/support.s index 5fa2aa7131fd..88a8b92b0df6 100644 --- a/sys/i386/i386/support.s +++ b/sys/i386/i386/support.s @@ -151,14 +151,15 @@ END(fillw) * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 */ ENTRY(bcopy) - pushl %ebp - movl %esp,%ebp - pushl %esi - pushl %edi - movl 8(%ebp),%esi - movl 12(%ebp),%edi - jmp 1f -ALTENTRY(memmove) + movl 4(%esp),%eax + movl 8(%esp),%edx + movl %eax,8(%esp) + movl %edx,4(%esp) + MEXITCOUNT + jmp memmove +END(bcopy) + +ENTRY(memmove) pushl %ebp movl %esp,%ebp pushl %esi @@ -208,7 +209,7 @@ ALTENTRY(memmove) movl 8(%ebp),%eax /* return dst for memmove */ popl %ebp ret -END(bcopy) +END(memmove) /* * Note: memcpy does not support overlapping copies @@ -463,13 +464,11 @@ END(handle_ibrs_entry) ENTRY(handle_ibrs_exit) cmpb $0,PCPU(IBPB_SET) je 1f - pushl %ecx movl $MSR_IA32_SPEC_CTRL,%ecx rdmsr andl $~(IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP),%eax andl $~((IA32_SPEC_CTRL_IBRS|IA32_SPEC_CTRL_STIBP)>>32),%edx wrmsr - popl %ecx movb $0,PCPU(IBPB_SET) 1: ret END(handle_ibrs_exit) diff --git a/sys/i386/include/asmacros.h b/sys/i386/include/asmacros.h index 8d478363ffc3..d558842d6b42 100644 --- a/sys/i386/include/asmacros.h +++ b/sys/i386/include/asmacros.h @@ -108,11 +108,11 @@ #define CROSSJUMPTARGET(label) \ ALIGN_TEXT; __CONCAT(to,label): ; MCOUNT; jmp label #define ENTRY(name) GEN_ENTRY(name) ; 9: ; MCOUNT -#define FAKE_MCOUNT(caller) pushl caller ; call __mcount ; popl %ecx -#define MCOUNT call __mcount +#define FAKE_MCOUNT(caller) pushl caller ; call *__mcountp ; popl %ecx +#define MCOUNT call *__mcountp #define MCOUNT_LABEL(name) GEN_ENTRY(name) ; nop ; ALIGN_TEXT #ifdef GUPROF -#define MEXITCOUNT call .mexitcount +#define MEXITCOUNT call *__mexitcountp #define ret MEXITCOUNT ; NON_GPROF_RET #else #define MEXITCOUNT diff --git a/sys/i386/include/profile.h b/sys/i386/include/profile.h index 5ebe287e1061..78b55a4463e1 100644 --- a/sys/i386/include/profile.h +++ b/sys/i386/include/profile.h @@ -92,16 +92,29 @@ extern int mcount_lock; void bintr(void); void btrap(void); void eintr(void); +#if 0 +void end_exceptions(void); +void start_exceptions(void); +#else +#include /* XXX */ +#endif void user(void); -#define MCOUNT_FROMPC_USER(pc) \ - ((pc < (uintfptr_t)VM_MAXUSER_ADDRESS) ? (uintfptr_t)user : pc) +#include /* XXX for setidt_disp */ + +#define MCOUNT_DETRAMP(pc) do { \ + if ((pc) >= (uintfptr_t)start_exceptions + setidt_disp && \ + (pc) < (uintfptr_t)end_exceptions + setidt_disp) \ + (pc) -= setidt_disp; \ +} while (0) #define MCOUNT_FROMPC_INTR(pc) \ ((pc >= (uintfptr_t)btrap && pc < (uintfptr_t)eintr) ? \ ((pc >= (uintfptr_t)bintr) ? (uintfptr_t)bintr : \ (uintfptr_t)btrap) : ~0U) +#define MCOUNT_USERPC ((uintfptr_t)user) + #else /* !_KERNEL */ #define FUNCTION_ALIGNMENT 4 diff --git a/sys/libkern/mcount.c b/sys/libkern/mcount.c index 8776aa347baa..2d387b96cc2d 100644 --- a/sys/libkern/mcount.c +++ b/sys/libkern/mcount.c @@ -88,13 +88,28 @@ _MCOUNT_DECL(uintfptr_t frompc, uintfptr_t selfpc) /* _mcount; may be static, in #endif #ifdef _KERNEL + /* De-relocate any addresses in a (single) trampoline. */ +#ifdef MCOUNT_DETRAMP + MCOUNT_DETRAMP(frompc); + MCOUNT_DETRAMP(selfpc); +#endif /* * When we are called from an exception handler, frompc may be * a user address. Convert such frompc's to some representation * in kernel address space. */ +#ifdef MCOUNT_FROMPC_USER frompc = MCOUNT_FROMPC_USER(frompc); +#elif defined(MCOUNT_USERPC) + /* + * For separate address spaces, we can only guess that addresses + * in the range known to us are actually kernel addresses. Outside + * of this range, conerting to the user address is fail-safe. + */ + if (frompc < p->lowpc || frompc - p->lowpc >= p->textsize) + frompc = MCOUNT_USERPC; #endif +#endif /* _KERNEL */ frompci = frompc - p->lowpc; if (frompci >= p->textsize) @@ -252,6 +267,9 @@ mexitcount(uintfptr_t selfpc) uintfptr_t selfpcdiff; p = &_gmonparam; +#ifdef MCOUNT_DETRAMP + MCOUNT_DETRAMP(selfpc); +#endif selfpcdiff = selfpc - (uintfptr_t)p->lowpc; if (selfpcdiff < p->textsize) { int delta;