amd64: partially depessimize cpu_fetch_syscall_args and cpu_set_syscall_retval

Vast majority of syscalls take 6 or less arguments. Move handling of other
cases to a fallback function. Similarly, special casing for _syscall
and __syscall
magic syscalls is moved away.

Return is almost always 0. The change replaces 3 branches with 1 in the common
case. Also the 'frame' variable convinces clang not to reload it on each access.

Reviewed by:	kib
Approved by:	re (gjb)
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D17542

sys/amd64/amd64/trap.c

@@ -970,21 +970,19 @@ dblfault_handler(struct trapframe *frame)
 	panic("double fault");
 }
 
-int
-cpu_fetch_syscall_args(struct thread *td)
+static int __noinline
+cpu_fetch_syscall_args_fallback(struct thread *td, struct syscall_args *sa)
 {
 	struct proc *p;
 	struct trapframe *frame;
 	register_t *argp;
-	struct syscall_args *sa;
 	caddr_t params;
 	int reg, regcnt, error;
 
 	p = td->td_proc;
 	frame = td->td_frame;
-	sa = &td->td_sa;
 	reg = 0;
-	regcnt = 6;
+	regcnt = NARGREGS;
 
 	sa->code = frame->tf_rax;
 
@@ -1002,24 +1000,58 @@ cpu_fetch_syscall_args(struct thread *td)
  		sa->callp = &p->p_sysent->sv_table[sa->code];
 
 	sa->narg = sa->callp->sy_narg;
-	KASSERT(sa->narg <= sizeof(sa->args) / sizeof(sa->args[0]),
-	    ("Too many syscall arguments!"));
-	error = 0;
+	KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
 	argp = &frame->tf_rdi;
 	argp += reg;
-	memcpy(sa->args, argp, sizeof(sa->args[0]) * 6);
+	memcpy(sa->args, argp, sizeof(sa->args[0]) * NARGREGS);
 	if (sa->narg > regcnt) {
 		params = (caddr_t)frame->tf_rsp + sizeof(register_t);
 		error = copyin(params, &sa->args[regcnt],
 	    	    (sa->narg - regcnt) * sizeof(sa->args[0]));
+		if (__predict_false(error != 0))
+			return (error);
 	}
 
-	if (error == 0) {
-		td->td_retval[0] = 0;
-		td->td_retval[1] = frame->tf_rdx;
-	}
+	td->td_retval[0] = 0;
+	td->td_retval[1] = frame->tf_rdx;
 
-	return (error);
+	return (0);
+}
+
+int
+cpu_fetch_syscall_args(struct thread *td)
+{
+	struct proc *p;
+	struct trapframe *frame;
+	struct syscall_args *sa;
+
+	p = td->td_proc;
+	frame = td->td_frame;
+	sa = &td->td_sa;
+
+	sa->code = frame->tf_rax;
+
+	if (__predict_false(sa->code == SYS_syscall ||
+	    sa->code == SYS___syscall ||
+	    sa->code >= p->p_sysent->sv_size))
+		return (cpu_fetch_syscall_args_fallback(td, sa));
+
+	sa->callp = &p->p_sysent->sv_table[sa->code];
+	sa->narg = sa->callp->sy_narg;
+	KASSERT(sa->narg <= nitems(sa->args), ("Too many syscall arguments!"));
+
+	if (p->p_sysent->sv_mask)
+		sa->code &= p->p_sysent->sv_mask;
+
+	if (__predict_false(sa->narg > NARGREGS))
+		return (cpu_fetch_syscall_args_fallback(td, sa));
+
+	memcpy(sa->args, &frame->tf_rdi, sizeof(sa->args[0]) * NARGREGS);
+
+	td->td_retval[0] = 0;
+	td->td_retval[1] = frame->tf_rdx;
+
+	return (0);
 }
 
 #include "../../kern/subr_syscall.c"

sys/amd64/amd64/vm_machdep.c

@@ -372,14 +372,17 @@ cpu_thread_free(struct thread *td)
 void
 cpu_set_syscall_retval(struct thread *td, int error)
 {
+	struct trapframe *frame;
+
+	frame = td->td_frame;
+	if (__predict_true(error == 0)) {
+		frame->tf_rax = td->td_retval[0];
+		frame->tf_rdx = td->td_retval[1];
+		frame->tf_rflags &= ~PSL_C;
+		return;
+	}
 
 	switch (error) {
-	case 0:
-		td->td_frame->tf_rax = td->td_retval[0];
-		td->td_frame->tf_rdx = td->td_retval[1];
-		td->td_frame->tf_rflags &= ~PSL_C;
-		break;
-
 	case ERESTART:
 		/*
 		 * Reconstruct pc, we know that 'syscall' is 2 bytes,
@@ -393,8 +396,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
 		 * Require full context restore to get the arguments
 		 * in the registers reloaded at return to usermode.
 		 */
-		td->td_frame->tf_rip -= td->td_frame->tf_err;
-		td->td_frame->tf_r10 = td->td_frame->tf_rcx;
+		frame->tf_rip -= frame->tf_err;
+		frame->tf_r10 = frame->tf_rcx;
 		set_pcb_flags(td->td_pcb, PCB_FULL_IRET);
 		break;
 
@@ -402,8 +405,8 @@ cpu_set_syscall_retval(struct thread *td, int error)
 		break;
 
 	default:
-		td->td_frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
-		td->td_frame->tf_rflags |= PSL_C;
+		frame->tf_rax = SV_ABI_ERRNO(td->td_proc, error);
+		frame->tf_rflags |= PSL_C;
 		break;
 	}
 }

sys/amd64/include/proc.h

@@ -101,6 +101,9 @@ int amd64_set_ldt_data(struct thread *td, int start, int num,
 
 extern struct mtx dt_lock;
 extern int max_ldt_segment;
+
+#define	NARGREGS	6
+
 #endif  /* _KERNEL */
 
 #endif /* !_MACHINE_PROC_H_ */