Create callable (non-inline) versions of the atomic_OP_TYPE functions

that are linked into the kernel.  The KLD compilation options are
changed to call these functions, rather than in-lining the
atomic operations.

This approach makes atomic operations from KLDs significantly
faster on UP systems (though somewhat slower on SMP systems).

PR:		i386/13111
Submitted by:	peter.jeremy@alcatel.com.au

sys/amd64/amd64/atomic.c

@@ -0,0 +1,47 @@
+/*-
+ * Copyright (c) 1999 Peter Jeremy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id $
+ */
+
+/* This file creates publically callable functions to perform various
+ * simple arithmetic on memory which is atomic in the presence of
+ * interrupts and multiple processors.
+ */
+#include <sys/types.h>
+
+/* Firstly make atomic.h generate prototypes as it will for kernel modules */
+#define KLD_MODULE
+#include <machine/atomic.h>
+#undef _MACHINE_ATOMIC_H_	/* forget we included it */
+#undef KLD_MODULE
+#undef ATOMIC_ASM
+
+/* Make atomic.h generate public functions */
+#define static
+#undef __inline
+#define __inline
+
+#include <machine/atomic.h>

sys/amd64/include/atomic.h

@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: atomic.h,v 1.3 1999/07/13 06:35:25 alc Exp $
+ *	$Id: atomic.h,v 1.4 1999/07/23 23:45:19 alc Exp $
  */
 #ifndef _MACHINE_ATOMIC_H_
 #define _MACHINE_ATOMIC_H_
@@ -54,9 +54,19 @@
  */
 
 /*
- * Make kernel modules portable between UP and SMP.
+ * The above functions are expanded inline in the statically-linked
+ * kernel.  Lock prefixes are generated if an SMP kernel is being
+ * built.
+ *
+ * Kernel modules call real functions which are built into the kernel.
+ * This allows kernel modules to be portable between UP and SMP systems.
  */
-#if defined(SMP) || defined(KLD_MODULE)
+#if defined(KLD_MODULE)
+#define ATOMIC_ASM(NAME, TYPE, OP, V)			\
+	extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v);
+
+#else /* !KLD_MODULE */
+#if defined(SMP)
 #define MPLOCKED	"lock ; "
 #else
 #define MPLOCKED
@@ -74,6 +84,7 @@ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 			 : "=m" (*p)			\
 			 :  "0" (*p), "ir" (V)); 	\
 }
+#endif /* KLD_MODULE */
 
 ATOMIC_ASM(set,	     char,  "orb %2,%0",   v)
 ATOMIC_ASM(clear,    char,  "andb %2,%0", ~v)

sys/conf/files.i386

@@ -1,7 +1,7 @@
 # This file tells config what files go into building a kernel,
 # files marked standard are always included.
 #
-#	$Id: files.i386,v 1.257 1999/08/07 12:19:41 peter Exp $
+#	$Id: files.i386,v 1.258 1999/08/09 10:34:38 phk Exp $
 #
 # The long compile-with and dependency lines are required because of
 # limitations in config: backslash-newline doesn't work in strings, and
@@ -117,6 +117,8 @@ i386/eisa/dpt_eisa.c		optional	eisa dpt
 i386/eisa/eisaconf.c		optional	eisa
 i386/eisa/if_fea.c		optional	fea
 i386/eisa/if_vx_eisa.c		optional	vx
+i386/i386/atomic.c		standard				\
+	compile-with	"${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} $<"
 i386/i386/autoconf.c		standard
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard

sys/i386/conf/files.i386

@@ -1,7 +1,7 @@
 # This file tells config what files go into building a kernel,
 # files marked standard are always included.
 #
-#	$Id: files.i386,v 1.257 1999/08/07 12:19:41 peter Exp $
+#	$Id: files.i386,v 1.258 1999/08/09 10:34:38 phk Exp $
 #
 # The long compile-with and dependency lines are required because of
 # limitations in config: backslash-newline doesn't work in strings, and
@@ -117,6 +117,8 @@ i386/eisa/dpt_eisa.c		optional	eisa dpt
 i386/eisa/eisaconf.c		optional	eisa
 i386/eisa/if_fea.c		optional	fea
 i386/eisa/if_vx_eisa.c		optional	vx
+i386/i386/atomic.c		standard				\
+	compile-with	"${CC} -c ${CFLAGS} ${DEFINED_PROF:S/^$/-fomit-frame-pointer/} $<"
 i386/i386/autoconf.c		standard
 i386/i386/bios.c		standard
 i386/i386/bioscall.s		standard

sys/i386/i386/atomic.c

@@ -0,0 +1,47 @@
+/*-
+ * Copyright (c) 1999 Peter Jeremy
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$Id $
+ */
+
+/* This file creates publically callable functions to perform various
+ * simple arithmetic on memory which is atomic in the presence of
+ * interrupts and multiple processors.
+ */
+#include <sys/types.h>
+
+/* Firstly make atomic.h generate prototypes as it will for kernel modules */
+#define KLD_MODULE
+#include <machine/atomic.h>
+#undef _MACHINE_ATOMIC_H_	/* forget we included it */
+#undef KLD_MODULE
+#undef ATOMIC_ASM
+
+/* Make atomic.h generate public functions */
+#define static
+#undef __inline
+#define __inline
+
+#include <machine/atomic.h>

sys/i386/include/atomic.h

@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: atomic.h,v 1.3 1999/07/13 06:35:25 alc Exp $
+ *	$Id: atomic.h,v 1.4 1999/07/23 23:45:19 alc Exp $
  */
 #ifndef _MACHINE_ATOMIC_H_
 #define _MACHINE_ATOMIC_H_
@@ -54,9 +54,19 @@
  */
 
 /*
- * Make kernel modules portable between UP and SMP.
+ * The above functions are expanded inline in the statically-linked
+ * kernel.  Lock prefixes are generated if an SMP kernel is being
+ * built.
+ *
+ * Kernel modules call real functions which are built into the kernel.
+ * This allows kernel modules to be portable between UP and SMP systems.
  */
-#if defined(SMP) || defined(KLD_MODULE)
+#if defined(KLD_MODULE)
+#define ATOMIC_ASM(NAME, TYPE, OP, V)			\
+	extern void atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v);
+
+#else /* !KLD_MODULE */
+#if defined(SMP)
 #define MPLOCKED	"lock ; "
 #else
 #define MPLOCKED
@@ -74,6 +84,7 @@ atomic_##NAME##_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\
 			 : "=m" (*p)			\
 			 :  "0" (*p), "ir" (V)); 	\
 }
+#endif /* KLD_MODULE */
 
 ATOMIC_ASM(set,	     char,  "orb %2,%0",   v)
 ATOMIC_ASM(clear,    char,  "andb %2,%0", ~v)