amd64: depessimize userspace memcpy/memmove/bcopy

The change resembles what was done in r334537 for kernel routines.
While here take care of i386 variants. Note that primitives remain
suboptimal.

Reviewed by:	kib (previous version)
Approved by:	re (gjb)
Differential Revision:	https://reviews.freebsd.org/D17167

lib/libc/amd64/string/bcopy.S

@@ -66,6 +66,9 @@ ENTRY(bcopy)
 	movsq
 	movq	%rdx,%rcx
 	andq	$7,%rcx		/* any bytes left? */
+	jne	2f
+	ret
+2:
 	rep
 	movsb
 	ret
@@ -73,11 +76,13 @@ ENTRY(bcopy)
 	addq	%rcx,%rdi	/* copy backwards. */
 	addq	%rcx,%rsi
 	std
-	andq	$7,%rcx		/* any fractional bytes? */
 	decq	%rdi
 	decq	%rsi
+	andq	$7,%rcx		/* any fractional bytes? */
+	je	3f
 	rep
 	movsb
+3:
 	movq	%rdx,%rcx	/* copy remainder by words */
 	shrq	$3,%rcx
 	subq	$7,%rsi

lib/libc/i386/string/bcopy.S

@@ -64,7 +64,7 @@ ENTRY(bcopy)
 	movl	%edi,%edx
 	subl	%esi,%edx
 	cmpl	%ecx,%edx	/* overlapping? */
-	jb	1f
+	jb	2f
 	cld			/* nope, copy forwards. */
 	movl	%ecx,%edx
 	shrl	$2,%ecx		/* copy by words */
@@ -72,21 +72,28 @@ ENTRY(bcopy)
 	movsl
 	movl	%edx,%ecx
 	andl	$3,%ecx		/* any bytes left? */
+	jne	1f
+	popl	%edi
+	popl	%esi
+	ret
+1:
 	rep
 	movsb
 	popl	%edi
 	popl	%esi
 	ret
-1:
+2:
 	addl	%ecx,%edi	/* copy backwards. */
 	addl	%ecx,%esi
 	std
 	movl	%ecx,%edx
-	andl	$3,%ecx		/* any fractional bytes? */
 	decl	%edi
 	decl	%esi
+	andl	$3,%ecx		/* any fractional bytes? */
+	je	3f
 	rep
 	movsb
+3:
 	movl	%edx,%ecx	/* copy remainder by words */
 	shrl	$2,%ecx
 	subl	$3,%esi