amd64: partially depessimize cpu_fetch_syscall_args and cpu_set_syscall_retval
Vast majority of syscalls take 6 or less arguments. Move handling of other cases to a fallback function. Similarly, special casing for _syscall and __syscall magic syscalls is moved away. Return is almost always 0. The change replaces 3 branches with 1 in the common case. Also the 'frame' variable convinces clang not to reload it on each access. Reviewed by: kib Approved by: re (gjb) Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D17542
This commit is contained in:
parent
9c7d70ee7d
commit
6816c88458
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=339349
@ -970,21 +970,19 @@ dblfault_handler(struct trapframe *frame)
|
||||
panic("double fault");
|
||||
}
|
||||
|
||||
int
|
||||
cpu_fetch_syscall_args(struct thread *td)
|
||||
static int __noinline
|
||||
cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
|
||||
{
|
||||
struct proc *p;
|
||||
struct trapframe *frame;
|
||||
register_t *argp;
|
||||
struct syscall_args *sa;
|
||||
caddr_t params;
|
||||
int reg, regcnt, error;
|
||||
|
||||
p = td->td_proc;
|
||||
frame = td->td_frame;
|
||||
sa = &td->td_sa;
|
||||
reg = 0;
|
||||
regcnt = 6;
|
||||
regcnt = NARGREGS;
|
||||
|
||||
sa->code = frame->tf_rax;
|
||||
|
||||
@ -1002,24 +1000,58 @@ cpu_fetch_syscall_args(struct thread *td)
|
||||
sa->callp = &p->p_sysent->sv_table[sa->code];
|
||||
|
||||
sa->narg = sa->callp->sy_narg;
|
||||
KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]),
|
||||
("Too many syscall arguments!"));
|
||||
error = 0;
|
||||
KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
|
||||
argp = &frame->tf_rdi;
|
||||
argp += reg;
|
||||
memcpy(sa->args, argp, sizeof(sa->args[0]) * 6);
|
||||
memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
|
||||
if (sa->narg > regcnt) {
|
||||
params = (caddr_t)frame->tf_rsp + sizeof(register_t);
|
||||
error = copyin(params, &sa->args[regcnt],
|
||||
(sa->narg - regcnt) * sizeof(sa->args[0]));
|
||||
if (__predict_false(error != 0))
|
||||
return (error);
|
||||
}
|
||||
|
||||
if (error == 0) {
|
||||
td->td_retval[0] = 0;
|
||||
td->td_retval[1] = frame->tf_rdx;
|
||||
}
|
||||
td->td_retval[0] = 0;
|
||||
td->td_retval[1] = frame->tf_rdx;
|
||||
|
||||
return (error);
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
cpu_fetch_syscall_args(struct thread *td)
|
||||
{
|
||||
struct proc *p;
|
||||
struct trapframe *frame;
|
||||
struct syscall_args *sa;
|
||||
|
||||
p = td->td_proc;
|
||||
frame = td->td_frame;
|
||||
sa = &td->td_sa;
|
||||
|
||||
sa->code = frame->tf_rax;
|
||||
|
||||
if (__predict_false(sa->code == SYS_syscall ||
|
||||
sa->code == SYS___syscall ||
|
||||
sa->code >= p->p_sysent->sv_size))
|
||||
return (cpu_fetch_syscall_args_fallback(td, sa));
|
||||
|
||||
sa->callp = &p->p_sysent->sv_table[sa->code];
|
||||
sa->narg = sa->callp->sy_narg;
|
||||
KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
|
||||
|
||||
if (p->p_sysent->sv_mask)
|
||||
sa->code &= p->p_sysent->sv_mask;
|
||||
|
||||
if (__predict_false(sa->narg > NARGREGS))
|
||||
return (cpu_fetch_syscall_args_fallback(td, sa));
|
||||
|
||||
memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
|
||||
|
||||
td->td_retval[0] = 0;
|
||||
td->td_retval[1] = frame->tf_rdx;
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
#include "../../kern/subr_syscall.c"
|
||||
|
@ -372,14 +372,17 @@ cpu_thread_free(struct thread *td)
|
||||
void
|
||||
cpu_set_syscall_retval(struct thread *td, int error)
|
||||
{
|
||||
struct trapframe *frame;
|
||||
|
||||
frame = td->td_frame;
|
||||
if (__predict_true(error == 0)) {
|
||||
frame->tf_rax = td->td_retval[0];
|
||||
frame->tf_rdx = td->td_retval[1];
|
||||
frame->tf_rflags &= ~PSL_C;
|
||||
return;
|
||||
}
|
||||
|
||||
switch (error) {
|
||||
case 0:
|
||||
td->td_frame->tf_rax = td->td_retval[0];
|
||||
td->td_frame->tf_rdx = td->td_retval[1];
|
||||
td->td_frame->tf_rflags &= ~PSL_C;
|
||||
break;
|
||||
|
||||
case ERESTART:
|
||||
/*
|
||||
* Reconstruct pc, we know that 'syscall' is 2 bytes,
|
||||
@ -393,8 +396,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
|
||||
* Require full context restore to get the arguments
|
||||
* in the registers reloaded at return to usermode.
|
||||
*/
|
||||
td->td_frame->tf_rip -= td->td_frame->tf_err;
|
||||
td->td_frame->tf_r10 = td->td_frame->tf_rcx;
|
||||
frame->tf_rip -= frame->tf_err;
|
||||
frame->tf_r10 = frame->tf_rcx;
|
||||
set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
|
||||
break;
|
||||
|
||||
@ -402,8 +405,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
|
||||
break;
|
||||
|
||||
default:
|
||||
td->td_frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
|
||||
td->td_frame->tf_rflags |= PSL_C;
|
||||
frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
|
||||
frame->tf_rflags |= PSL_C;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
@ -101,6 +101,9 @@ int amd64_set_ldt_data(struct thread *td, int start, int num,
|
||||
|
||||
extern struct mtx dt_lock;
|
||||
extern int max_ldt_segment;
|
||||
|
||||
#define NARGREGS 6
|
||||
|
||||
#endif /* _KERNEL */
|
||||
|
||||
#endif /* !_MACHINE_PROC_H_ */
|
||||
|
Loading…
Reference in New Issue
Block a user