amd64: even up copyin/copyout with memcpy + other cleanup

- _fault handlers for both primitives are identical, provide just one
- change the copying scheme to match memcpy (in particular jump
avoidance for the most common case of multiply of 8)
- stop re-reading pcb address on exit, just store it locally (in r9)

Reviewed by:	kib
Approved by:	re (gjb)
Differential Revision:	https://reviews.freebsd.org/D17265

sys/amd64/amd64/support.S

@@ -309,9 +309,9 @@ END(fillw)
  */
 .macro	COPYOUT smap erms
 	PUSH_FRAME_POINTER
-	movq	PCPU(CURPCB),%rax
+	movq	PCPU(CURPCB),%r9
 	/* Trap entry clears PSL.AC */
-	movq	$copyout_fault,PCB_ONFAULT(%rax)
+	movq	$copy_fault,PCB_ONFAULT(%r9)
 	testq	%rdx,%rdx			/* anything to do? */
 	jz	2f
 
@@ -327,7 +327,7 @@ END(fillw)
 	 */
 	movq	%rsi,%rax
 	addq	%rdx,%rax
-	jc	copyout_fault
+	jc	copy_fault
 /*
  * XXX STOP USING VM_MAXUSER_ADDRESS.
  * It is an end address, not a max, so every time it is used correctly it
@@ -336,7 +336,7 @@ END(fillw)
  */
 	movq	$VM_MAXUSER_ADDRESS,%rcx
 	cmpq	%rcx,%rax
-	ja	copyout_fault
+	ja	copy_fault
 
 	xchgq	%rdi,%rsi
 	/* bcopy(%rsi, %rdi, %rdx) */
@@ -344,21 +344,27 @@ END(fillw)
 
 	SMAP_DISABLE \smap
 .if	\erms == 0
+	cmpq	$15,%rcx
+	jbe	1f
 	shrq	$3,%rcx
 	rep
 	movsq
 	movb	%dl,%cl
 	andb	$7,%cl
-	je	1f
+	jne	1f
+	SMAP_ENABLE \smap
+	xorl	%eax,%eax
+	movq	%rax,PCB_ONFAULT(%r9)
+	POP_FRAME_POINTER
+	ret
 .endif
+1:
 	rep
 	movsb
-1:
 	SMAP_ENABLE \smap
 2:
 	xorl	%eax,%eax
-	movq	PCPU(CURPCB),%rdx
-	movq	%rax,PCB_ONFAULT(%rdx)
+	movq	%rax,PCB_ONFAULT(%r9)
 	POP_FRAME_POINTER
 	ret
 .endmacro
@@ -379,22 +385,14 @@ ENTRY(copyout_smap_erms)
 	COPYOUT smap=1 erms=1
 END(copyout_smap_erms)
 
-	ALIGN_TEXT
-copyout_fault:
-	movq	PCPU(CURPCB),%rdx
-	movq	$0,PCB_ONFAULT(%rdx)
-	movq	$EFAULT,%rax
-	POP_FRAME_POINTER
-	ret
-
 /*
  * copyin(from_user, to_kernel, len)
  *        %rdi,      %rsi,      %rdx
  */
 .macro	COPYIN smap erms
 	PUSH_FRAME_POINTER
-	movq	PCPU(CURPCB),%rax
-	movq	$copyin_fault,PCB_ONFAULT(%rax)
+	movq	PCPU(CURPCB),%r9
+	movq	$copy_fault,PCB_ONFAULT(%r9)
 	testq	%rdx,%rdx			/* anything to do? */
 	jz	2f
 
@@ -403,10 +401,10 @@ copyout_fault:
 	 */
 	movq	%rdi,%rax
 	addq	%rdx,%rax
-	jc	copyin_fault
+	jc	copy_fault
 	movq	$VM_MAXUSER_ADDRESS,%rcx
 	cmpq	%rcx,%rax
-	ja	copyin_fault
+	ja	copy_fault
 
 	xchgq	%rdi,%rsi
 	movq	%rdx,%rcx
@@ -414,22 +412,28 @@ copyout_fault:
 
 	SMAP_DISABLE \smap
 .if \erms == 0
+	cmpq	$15,%rcx
+	jbe	1f
 	shrq	$3,%rcx				/* copy longword-wise */
 	rep
 	movsq
 	movb	%al,%cl
 	andb	$7,%cl				/* copy remaining bytes */
-	je	1f
+	jne	1f
+	SMAP_ENABLE \smap
+	xorl	%eax,%eax
+	movq	%rax,PCB_ONFAULT(%r9)
+	POP_FRAME_POINTER
+	ret
 .endif
+1:
 	rep
 	movsb
 
-1:
 	SMAP_ENABLE \smap
 2:
 	xorl	%eax,%eax
-	movq	PCPU(CURPCB),%rdx
-	movq	%rax,PCB_ONFAULT(%rdx)
+	movq	%rax,PCB_ONFAULT(%r9)
 	POP_FRAME_POINTER
 	ret
 .endmacro
@@ -451,10 +455,10 @@ ENTRY(copyin_smap_erms)
 END(copyin_smap_erms)
 
 	ALIGN_TEXT
-copyin_fault:
+copy_fault:
 	movq	PCPU(CURPCB),%rdx
 	movq	$0,PCB_ONFAULT(%rdx)
-	movq	$EFAULT,%rax
+	movl	$EFAULT,%eax
 	POP_FRAME_POINTER
 	ret