From 91c09a383ab76991061b49c3e081c06b3da2a98c Mon Sep 17 00:00:00 2001 From: Alan Cox Date: Thu, 7 Apr 2005 03:56:03 +0000 Subject: [PATCH] Add machine-specific, optimized implementations of bcopy, bzero, memcpy, memmove, and memset. PR: 73111 Submitted by: Ville-Pertti Keinonen (taken from NetBSD) MFC after: 3 weeks --- lib/libc/amd64/string/Makefile.inc | 3 + lib/libc/amd64/string/bcopy.S | 94 ++++++++++++++++++++++++++++++ lib/libc/amd64/string/bzero.S | 45 ++++++++++++++ lib/libc/amd64/string/memcpy.S | 5 ++ lib/libc/amd64/string/memmove.S | 5 ++ lib/libc/amd64/string/memset.S | 60 +++++++++++++++++++ 6 files changed, 212 insertions(+) create mode 100644 lib/libc/amd64/string/Makefile.inc create mode 100644 lib/libc/amd64/string/bcopy.S create mode 100644 lib/libc/amd64/string/bzero.S create mode 100644 lib/libc/amd64/string/memcpy.S create mode 100644 lib/libc/amd64/string/memmove.S create mode 100644 lib/libc/amd64/string/memset.S diff --git a/lib/libc/amd64/string/Makefile.inc b/lib/libc/amd64/string/Makefile.inc new file mode 100644 index 000000000000..725339fd5bb8 --- /dev/null +++ b/lib/libc/amd64/string/Makefile.inc @@ -0,0 +1,3 @@ +# $FreeBSD$ + +MDSRCS+= bcopy.S bzero.S memcpy.S memmove.S memset.S diff --git a/lib/libc/amd64/string/bcopy.S b/lib/libc/amd64/string/bcopy.S new file mode 100644 index 000000000000..8289e968bab8 --- /dev/null +++ b/lib/libc/amd64/string/bcopy.S @@ -0,0 +1,94 @@ +/*- + * Copyright (c) 1990 The Regents of the University of California. + * All rights reserved. + * + * This code is derived from locore.s. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#if 0 + RCSID("$NetBSD: bcopy.S,v 1.2 2003/08/07 16:42:36 agc Exp $") +#endif + + /* + * (ov)bcopy (src,dst,cnt) + * ws@tools.de (Wolfgang Solfrank, TooLs GmbH) +49-228-985800 + */ + +#ifdef MEMCOPY +ENTRY(memcpy) +#else +#ifdef MEMMOVE +ENTRY(memmove) +#else +ENTRY(bcopy) +#endif +#endif +#if defined(MEMCOPY) || defined(MEMMOVE) + movq %rdi,%r11 /* save dest */ +#else + xchgq %rdi,%rsi +#endif + movq %rdx,%rcx + movq %rdi,%rax + subq %rsi,%rax + cmpq %rcx,%rax /* overlapping? */ + jb 1f + cld /* nope, copy forwards. */ + shrq $3,%rcx /* copy by words */ + rep + movsq + movq %rdx,%rcx + andq $7,%rcx /* any bytes left? */ + rep + movsb +#if defined(MEMCOPY) || defined(MEMMOVE) + movq %r11,%rax +#endif + ret +1: + addq %rcx,%rdi /* copy backwards. */ + addq %rcx,%rsi + std + andq $7,%rcx /* any fractional bytes? */ + decq %rdi + decq %rsi + rep + movsb + movq %rdx,%rcx /* copy remainder by words */ + shrq $3,%rcx + subq $7,%rsi + subq $7,%rdi + rep + movsq +#if defined(MEMCOPY) || defined(MEMMOVE) + movq %r11,%rax +#endif + cld + ret diff --git a/lib/libc/amd64/string/bzero.S b/lib/libc/amd64/string/bzero.S new file mode 100644 index 000000000000..2f4ced3c0423 --- /dev/null +++ b/lib/libc/amd64/string/bzero.S @@ -0,0 +1,45 @@ +/* + * Written by J.T. Conklin . + * Public domain. + * Adapted for NetBSD/x86_64 by Frank van der Linden + */ + +#include +__FBSDID("$FreeBSD$"); + +#if 0 + RCSID("$NetBSD: bzero.S,v 1.2 2003/07/26 19:24:38 salo Exp $") +#endif + +ENTRY(bzero) + movq %rsi,%rdx + + cld /* set fill direction forward */ + xorq %rax,%rax /* set fill data to 0 */ + + /* + * if the string is too short, it's really not worth the overhead + * of aligning to word boundries, etc. So we jump to a plain + * unaligned set. + */ + cmpq $16,%rdx + jb L1 + + movq %rdi,%rcx /* compute misalignment */ + negq %rcx + andq $7,%rcx + subq %rcx,%rdx + rep /* zero until word aligned */ + stosb + + movq %rdx,%rcx /* zero by words */ + shrq $3,%rcx + andq $7,%rdx + rep + stosq + +L1: movq %rdx,%rcx /* zero remainder by bytes */ + rep + stosb + + ret diff --git a/lib/libc/amd64/string/memcpy.S b/lib/libc/amd64/string/memcpy.S new file mode 100644 index 000000000000..bd1e8422ad84 --- /dev/null +++ b/lib/libc/amd64/string/memcpy.S @@ -0,0 +1,5 @@ +/* $NetBSD: memcpy.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */ +/* $FreeBSD$ */ + +#define MEMCOPY +#include "bcopy.S" diff --git a/lib/libc/amd64/string/memmove.S b/lib/libc/amd64/string/memmove.S new file mode 100644 index 000000000000..85beb262f8d9 --- /dev/null +++ b/lib/libc/amd64/string/memmove.S @@ -0,0 +1,5 @@ +/* $NetBSD: memmove.S,v 1.1 2001/06/19 00:25:05 fvdl Exp $ */ +/* $FreeBSD$ */ + +#define MEMMOVE +#include "bcopy.S" diff --git a/lib/libc/amd64/string/memset.S b/lib/libc/amd64/string/memset.S new file mode 100644 index 000000000000..142387aebd86 --- /dev/null +++ b/lib/libc/amd64/string/memset.S @@ -0,0 +1,60 @@ +/* + * Written by J.T. Conklin . + * Public domain. + * Adapted for NetBSD/x86_64 by Frank van der Linden + */ + +#include +__FBSDID("$FreeBSD$"); + +#if 0 + RCSID("$NetBSD: memset.S,v 1.3 2004/02/26 20:50:06 drochner Exp $") +#endif + +ENTRY(memset) + movq %rsi,%rax + andq $0xff,%rax + movq %rdx,%rcx + movq %rdi,%r11 + + cld /* set fill direction forward */ + + /* + * if the string is too short, it's really not worth the overhead + * of aligning to word boundries, etc. So we jump to a plain + * unaligned set. + */ + cmpq $0x0f,%rcx + jle L1 + + movb %al,%ah /* copy char to all bytes in word */ + movl %eax,%edx + sall $16,%eax + orl %edx,%eax + + movl %eax,%edx + salq $32,%rax + orq %rdx,%rax + + movq %rdi,%rdx /* compute misalignment */ + negq %rdx + andq $7,%rdx + movq %rcx,%r8 + subq %rdx,%r8 + + movq %rdx,%rcx /* set until word aligned */ + rep + stosb + + movq %r8,%rcx + shrq $3,%rcx /* set by words */ + rep + stosq + + movq %r8,%rcx /* set remainder by bytes */ + andq $7,%rcx +L1: rep + stosb + movq %r11,%rax + + ret