make critical_{enter, exit} inline
Avoid pulling in all of the <sys/proc.h> dependencies by automatically generating a stripped down thread_lite exporting only the fields of interest. The field declarations are type checked against the original and the offsets of the generated result is automatically checked. kib has expressed disagreement and would have preferred to simply use genassym style offsets (which loses type check enforcement). jhb has expressed dislike of it due to header pollution and a duplicate structure. He would have preferred to just have defined thread in _thread.h. Nonetheless, he admits that this is the only viable solution at the moment. The impetus for this came from mjg's D15331: "Inline critical_enter/exit for amd64" Reviewed by: jeff Differential Revision: https://reviews.freebsd.org/D16078
This commit is contained in:
Matt Macy
parent
28acc09dc0
commit
6443773dab
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=335879
@ -185,13 +185,25 @@ hack.pico: Makefile
|
||||
${CC} ${HACK_EXTRA_FLAGS} -nostdlib hack.c -o hack.pico
|
||||
rm -f hack.c
|
||||
|
||||
assym.inc: $S/kern/genassym.sh genassym.o
|
||||
offset.inc: $S/kern/genoffset.sh genoffset.o
|
||||
NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genoffset.sh genoffset.o > ${.TARGET}
|
||||
|
||||
genoffset.o: $S/kern/genoffset.c
|
||||
${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/kern/genoffset.c
|
||||
|
||||
genoffset_test.c: $S/kern/genoffset.c
|
||||
cp $S/kern/genoffset.c genoffset_test.c
|
||||
|
||||
genoffset_test.o: genoffset_test.c offset.inc
|
||||
${CC} -c ${CFLAGS:N-flto:N-fno-common} -DOFFSET_TEST genoffset_test.c
|
||||
|
||||
assym.inc: $S/kern/genassym.sh genassym.o genoffset_test.o
|
||||
NM='${NM}' NMFLAGS='${NMFLAGS}' sh $S/kern/genassym.sh genassym.o > ${.TARGET}
|
||||
|
||||
genassym.o: $S/$M/$M/genassym.c
|
||||
genassym.o: $S/$M/$M/genassym.c offset.inc
|
||||
${CC} -c ${CFLAGS:N-flto:N-fno-common} $S/$M/$M/genassym.c
|
||||
|
||||
${SYSTEM_OBJS} genassym.o vers.o: opt_global.h
|
||||
${SYSTEM_OBJS} genoffset.o genassym.o vers.o: opt_global.h
|
||||
|
||||
.if !empty(.MAKE.MODE:Unormal:Mmeta) && empty(.MAKE.MODE:Unormal:Mnofilemon)
|
||||
_meta_filemon= 1
|
||||
@ -213,10 +225,10 @@ _SKIP_DEPEND= 1
|
||||
.endif
|
||||
|
||||
kernel-depend: .depend
|
||||
SRCS= assym.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \
|
||||
SRCS= assym.inc offset.inc vnode_if.h ${BEFORE_DEPEND} ${CFILES} \
|
||||
${SYSTEM_CFILES} ${GEN_CFILES} ${SFILES} \
|
||||
${MFILES:T:S/.m$/.h/}
|
||||
DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o
|
||||
DEPENDOBJS+= ${SYSTEM_OBJS} genassym.o genoffset.o
|
||||
DEPENDFILES= ${DEPENDOBJS:O:u:C/^/.depend./}
|
||||
.if ${MAKE_VERSION} < 20160220
|
||||
DEPEND_MP?= -MP
|
||||
|
||||
@ -195,7 +195,7 @@ OFEDCFLAGS= ${CFLAGS:N-I*} -DCONFIG_INFINIBAND_USER_MEM \
|
||||
OFED_C_NOIMP= ${CC} -c -o ${.TARGET} ${OFEDCFLAGS} ${WERROR} ${PROF}
|
||||
OFED_C= ${OFED_C_NOIMP} ${.IMPSRC}
|
||||
|
||||
GEN_CFILES= $S/$M/$M/genassym.c ${MFILES:T:S/.m$/.c/}
|
||||
GEN_CFILES= $S/$M/$M/genassym.c $S/kern/genoffset.c ${MFILES:T:S/.m$/.c/}
|
||||
SYSTEM_CFILES= config.c env.c hints.c vnode_if.c
|
||||
SYSTEM_DEP= Makefile ${SYSTEM_OBJS}
|
||||
SYSTEM_OBJS= locore.o ${MDOBJS} ${OBJS}
|
||||
|
||||
44
sys/kern/genoffset.c
Normal file
44
sys/kern/genoffset.c
Normal file
@ -0,0 +1,44 @@
|
||||
/*-
|
||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
||||
*
|
||||
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
#ifndef OFFSET_TEST
|
||||
#define GENOFFSET
|
||||
#endif
|
||||
#include <sys/cdefs.h>
|
||||
__FBSDID("$FreeBSD$");
|
||||
#include <sys/param.h>
|
||||
#include <sys/assym.h>
|
||||
#include <sys/proc.h>
|
||||
|
||||
OFFSYM(td_pre_epoch_prio, thread, u_char);
|
||||
OFFSYM(td_priority, thread, u_char);
|
||||
OFFSYM(td_epochnest, thread, u_char);
|
||||
OFFSYM(td_critnest, thread, u_int);
|
||||
OFFSYM(td_pinned, thread, int);
|
||||
OFFSYM(td_owepreempt, thread, u_char);
|
||||
141
sys/kern/genoffset.sh
Normal file
141
sys/kern/genoffset.sh
Normal file
@ -0,0 +1,141 @@
|
||||
#!/bin/sh
|
||||
|
||||
# SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
||||
#
|
||||
# Copyright (c) 2000, Bruce Evans <bde@freebsd.org>
|
||||
# Copyright (c) 2018, Jeff Roberson <jeff@freebsd.org>
|
||||
#
|
||||
# Redistribution and use in source and binary forms, with or without
|
||||
# modification, are permitted provided that the following conditions
|
||||
# are met:
|
||||
# 1. Redistributions of source code must retain the above copyright
|
||||
# notice, this list of conditions and the following disclaimer.
|
||||
# 2. Redistributions in binary form must reproduce the above copyright
|
||||
# notice, this list of conditions and the following disclaimer in the
|
||||
# documentation and/or other materials provided with the distribution.
|
||||
#
|
||||
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
# SUCH DAMAGE.
|
||||
#
|
||||
# $FreeBSD$
|
||||
|
||||
usage()
|
||||
{
|
||||
echo "usage: genoffset [-o outfile] objfile"
|
||||
exit 1
|
||||
}
|
||||
|
||||
|
||||
work()
|
||||
{
|
||||
echo "#ifndef _OFFSET_INC_"
|
||||
echo "#define _OFFSET_INC_"
|
||||
echo "#if !defined(GENOFFSET) && !defined(KLD_MODULE)"
|
||||
${NM:='nm'} ${NMFLAGS} "$1" | ${AWK:='awk'} '
|
||||
/ C .*_datatype_*/ {
|
||||
type = substr($3, match($3, "_datatype_") + length("_datatype_"))
|
||||
}
|
||||
/ C .*_parenttype_*/ {
|
||||
parent = substr($3, match($3, "_parenttype_") + length("_parenttype_"))
|
||||
}
|
||||
/ C .*sign$/ {
|
||||
sign = substr($1, length($1) - 3, 4)
|
||||
sub("^0*", "", sign)
|
||||
if (sign != "")
|
||||
sign = "-"
|
||||
}
|
||||
/ C .*w0$/ {
|
||||
w0 = substr($1, length($1) - 3, 4)
|
||||
}
|
||||
/ C .*w1$/ {
|
||||
w1 = substr($1, length($1) - 3, 4)
|
||||
}
|
||||
/ C .*w2$/ {
|
||||
w2 = substr($1, length($1) - 3, 4)
|
||||
}
|
||||
/ C .*w3$/ {
|
||||
w3 = substr($1, length($1) - 3, 4)
|
||||
w = w3 w2 w1 w0
|
||||
sub("^0*", "", w)
|
||||
if (w == "")
|
||||
w = "0"
|
||||
hex = ""
|
||||
if (w != "0")
|
||||
hex = "0x"
|
||||
sub("w3$", "", $3)
|
||||
member = tolower($3)
|
||||
# This still has minor problems representing INT_MIN, etc.
|
||||
# E.g.,
|
||||
# with 32-bit 2''s complement ints, this prints -0x80000000,
|
||||
# which has the wrong type (unsigned int).
|
||||
offset = sprintf("%s%s%s", sign, hex, w)
|
||||
|
||||
structures[parent] = sprintf("%s%s %s %s\n",
|
||||
structures[parent], offset, type, member)
|
||||
}
|
||||
END {
|
||||
for (struct in structures) {
|
||||
printf("struct %s_lite {\n", struct);
|
||||
n = split(structures[struct], members, "\n")
|
||||
for (i = 1; i < n; i++) {
|
||||
for (j = i + 1; j < n; j++) {
|
||||
split(members[i], ivar, " ")
|
||||
split(members[j], jvar, " ")
|
||||
if (jvar[1] < ivar[1]) {
|
||||
tmp = members[i]
|
||||
members[i] = members[j]
|
||||
members[j] = tmp
|
||||
}
|
||||
}
|
||||
}
|
||||
off = "0"
|
||||
for (i = 1; i < n; i++) {
|
||||
split(members[i], m, " ")
|
||||
printf "\tu_char\tpad_%s[%s - %s];\n", m[3], m[1], off
|
||||
printf "\t%s\t%s;\n", m[2], m[3]
|
||||
off = sprintf("(%s + sizeof(%s))", m[1], m[2])
|
||||
}
|
||||
printf("};\n");
|
||||
}
|
||||
}
|
||||
'
|
||||
|
||||
echo "#endif"
|
||||
echo "#endif"
|
||||
}
|
||||
|
||||
|
||||
#
|
||||
#MAIN PROGGRAM
|
||||
#
|
||||
use_outfile="no"
|
||||
while getopts "o:" option
|
||||
do
|
||||
case "$option" in
|
||||
o) outfile="$OPTARG"
|
||||
use_outfile="yes";;
|
||||
*) usage;;
|
||||
esac
|
||||
done
|
||||
shift $(($OPTIND - 1))
|
||||
case $# in
|
||||
1) ;;
|
||||
*) usage;;
|
||||
esac
|
||||
|
||||
if [ "$use_outfile" = "yes" ]
|
||||
then
|
||||
work $1 3>"$outfile" >&3 3>&-
|
||||
else
|
||||
work $1
|
||||
fi
|
||||
|
||||
@ -199,17 +199,17 @@ choosethread(void)
|
||||
* the function call itself, for most cases.
|
||||
*/
|
||||
void
|
||||
critical_enter(void)
|
||||
critical_enter_KBI(void)
|
||||
{
|
||||
struct thread *td;
|
||||
|
||||
td = curthread;
|
||||
td->td_critnest++;
|
||||
#ifdef KTR
|
||||
struct thread *td = curthread;
|
||||
#endif
|
||||
critical_enter();
|
||||
CTR4(KTR_CRITICAL, "critical_enter by thread %p (%ld, %s) to %d", td,
|
||||
(long)td->td_proc->p_pid, td->td_name, td->td_critnest);
|
||||
}
|
||||
|
||||
static void __noinline
|
||||
void __noinline
|
||||
critical_exit_preempt(void)
|
||||
{
|
||||
struct thread *td;
|
||||
@ -245,17 +245,12 @@ critical_exit_preempt(void)
|
||||
}
|
||||
|
||||
void
|
||||
critical_exit(void)
|
||||
critical_exit_KBI(void)
|
||||
{
|
||||
struct thread *td;
|
||||
|
||||
td = curthread;
|
||||
KASSERT(td->td_critnest != 0,
|
||||
("critical_exit: td_critnest == 0"));
|
||||
td->td_critnest--;
|
||||
__compiler_membar();
|
||||
if (__predict_false(td->td_owepreempt))
|
||||
critical_exit_preempt();
|
||||
#ifdef KTR
|
||||
struct thread *td = curthread;
|
||||
#endif
|
||||
critical_exit();
|
||||
CTR4(KTR_CRITICAL, "critical_exit by thread %p (%ld, %s) to %d", td,
|
||||
(long)td->td_proc->p_pid, td->td_name, td->td_critnest);
|
||||
}
|
||||
|
||||
@ -43,4 +43,19 @@ char name ## w1[((ASSYM_ABS(value) & 0xFFFF0000UL) >> 16) + ASSYM_BIAS]; \
|
||||
char name ## w2[((ASSYM_ABS(value) & 0xFFFF00000000ULL) >> 32) + ASSYM_BIAS]; \
|
||||
char name ## w3[((ASSYM_ABS(value) & 0xFFFF000000000000ULL) >> 48) + ASSYM_BIAS]
|
||||
|
||||
|
||||
/* char name ## _datatype_ ## STRINGIFY(typeof(((struct parenttype *)(0x0))-> name)) [1]; */
|
||||
#ifdef OFFSET_TEST
|
||||
#define OFFSET_CTASSERT CTASSERT
|
||||
#else
|
||||
#define OFFSET_CTASSERT(...)
|
||||
#endif
|
||||
|
||||
#define OFFSYM(name, parenttype, datatype) \
|
||||
ASSYM(name, offsetof(struct parenttype, name)); \
|
||||
char name ## _datatype_ ## datatype [1]; \
|
||||
char name ## _parenttype_ ## parenttype [1]; \
|
||||
CTASSERT(__builtin_types_compatible_p(__typeof(((struct parenttype *)(0x0))-> name), datatype)); \
|
||||
OFFSET_CTASSERT(offsetof(struct parenttype, name) == offsetof(struct parenttype ## _lite, name))
|
||||
|
||||
#endif /* !_SYS_ASSYM_H_ */
|
||||
|
||||
55
sys/sys/kpilite.h
Normal file
55
sys/sys/kpilite.h
Normal file
@ -0,0 +1,55 @@
|
||||
/*-
|
||||
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
|
||||
*
|
||||
* Copyright (c) 2018, Matthew Macy <mmacy@freebsd.org>
|
||||
*
|
||||
* Redistribution and use in source and binary forms, with or without
|
||||
* modification, are permitted provided that the following conditions
|
||||
* are met:
|
||||
* 1. Redistributions of source code must retain the above copyright
|
||||
* notice, this list of conditions and the following disclaimer.
|
||||
* 2. Redistributions in binary form must reproduce the above copyright
|
||||
* notice, this list of conditions and the following disclaimer in the
|
||||
* documentation and/or other materials provided with the distribution.
|
||||
*
|
||||
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
||||
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
||||
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
||||
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
||||
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
||||
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
||||
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
||||
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
||||
* SUCH DAMAGE.
|
||||
*
|
||||
* $FreeBSD$
|
||||
*/
|
||||
|
||||
#ifndef _SYS_KPILITE_H_
|
||||
#define _SYS_KPILITE_H_
|
||||
#if !defined(GENOFFSET) && !defined(KLD_MODULE) && defined(_KERNEL)
|
||||
#include "offset.inc"
|
||||
|
||||
static __inline void
|
||||
sched_pin_lite(struct thread_lite *td)
|
||||
{
|
||||
|
||||
KASSERT((struct thread *)td == curthread, ("sched_pin called on non curthread"));
|
||||
td->td_pinned++;
|
||||
__compiler_membar();
|
||||
}
|
||||
|
||||
static __inline void
|
||||
sched_unpin_lite(struct thread_lite *td)
|
||||
{
|
||||
|
||||
KASSERT((struct thread *)td == curthread, ("sched_unpin called on non curthread"));
|
||||
KASSERT(td->td_pinned > 0, ("sched_unpin called on non pinned thread"));
|
||||
__compiler_membar();
|
||||
td->td_pinned--;
|
||||
__compiler_membar();
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@ -107,6 +107,12 @@ void kassert_panic(const char *fmt, ...) __printflike(1, 2);
|
||||
#define CTASSERT(x) _Static_assert(x, "compile-time assertion failed")
|
||||
#endif
|
||||
|
||||
#if defined(_KERNEL)
|
||||
#include <sys/param.h> /* MAXCPU */
|
||||
#include <sys/pcpu.h> /* curthread */
|
||||
#include <sys/kpilite.h>
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Assert that a pointer can be loaded from memory atomically.
|
||||
*
|
||||
@ -214,12 +220,44 @@ void vpanic(const char *, __va_list) __dead2 __printflike(1, 0);
|
||||
void cpu_boot(int);
|
||||
void cpu_flush_dcache(void *, size_t);
|
||||
void cpu_rootconf(void);
|
||||
void critical_enter(void);
|
||||
void critical_exit(void);
|
||||
void critical_enter_KBI(void);
|
||||
void critical_exit_KBI(void);
|
||||
void critical_exit_preempt(void);
|
||||
void init_param1(void);
|
||||
void init_param2(long physpages);
|
||||
void init_static_kenv(char *, size_t);
|
||||
void tablefull(const char *);
|
||||
|
||||
#if defined(KLD_MODULE) || defined(KTR_CRITICAL) || !defined(_KERNEL) || defined(GENOFFSET)
|
||||
#define critical_enter() critical_enter_KBI()
|
||||
#define critical_exit() critical_exit_KBI()
|
||||
#else
|
||||
static __inline void
|
||||
critical_enter(void)
|
||||
{
|
||||
struct thread_lite *td;
|
||||
|
||||
td = (struct thread_lite *)curthread;
|
||||
td->td_critnest++;
|
||||
}
|
||||
|
||||
static __inline void
|
||||
critical_exit(void)
|
||||
{
|
||||
struct thread_lite *td;
|
||||
|
||||
td = (struct thread_lite *)curthread;
|
||||
KASSERT(td->td_critnest != 0,
|
||||
("critical_exit: td_critnest == 0"));
|
||||
td->td_critnest--;
|
||||
__compiler_membar();
|
||||
if (__predict_false(td->td_owepreempt))
|
||||
critical_exit_preempt();
|
||||
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef EARLY_PRINTF
|
||||
typedef void early_putc_t(int ch);
|
||||
extern early_putc_t *early_putc;
|
||||
|
||||
Loading…
Reference in New Issue
Block a user