Introduce curpcb magic variable, similar to curthread, which is MD
amd64. It is implemented as __pure2 inline with non-volatile asm read from pcpu, which allows a compiler to cache its results. Convert most PCPU_GET(pcb) and curthread->td_pcb accesses into curpcb. Note that __curthread() uses magic value 0 as an offsetof(struct pcpu, pc_curthread). It seems to be done this way due to machine/pcpu.h needs to be processed before sys/pcpu.h, because machine/pcpu.h contributes machine-depended fields to the struct pcpu definition. As result, machine/pcpu.h cannot use struct pcpu yet. The __curpcb() also uses a magic constant instead of offsetof(struct pcpu, pc_curpcb) for the same reason. The constants are now defined as symbols and CTASSERTs are added to ensure that future KBI changes do not break the code. Requested and reviewed by: bde MFC after: 3 weeks
This commit is contained in:
parent
78cf63fc10
commit
83b22b05e6
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=238623
@ -327,7 +327,7 @@ fpuexit(struct thread *td)
|
|||||||
critical_enter();
|
critical_enter();
|
||||||
if (curthread == PCPU_GET(fpcurthread)) {
|
if (curthread == PCPU_GET(fpcurthread)) {
|
||||||
stop_emulating();
|
stop_emulating();
|
||||||
fpusave(PCPU_GET(curpcb)->pcb_save);
|
fpusave(curpcb->pcb_save);
|
||||||
start_emulating();
|
start_emulating();
|
||||||
PCPU_SET(fpcurthread, 0);
|
PCPU_SET(fpcurthread, 0);
|
||||||
}
|
}
|
||||||
@ -547,7 +547,7 @@ fputrap_x87(void)
|
|||||||
* wherever they are.
|
* wherever they are.
|
||||||
*/
|
*/
|
||||||
if (PCPU_GET(fpcurthread) != curthread) {
|
if (PCPU_GET(fpcurthread) != curthread) {
|
||||||
pcb_save = PCPU_GET(curpcb)->pcb_save;
|
pcb_save = curpcb->pcb_save;
|
||||||
control = pcb_save->sv_env.en_cw;
|
control = pcb_save->sv_env.en_cw;
|
||||||
status = pcb_save->sv_env.en_sw;
|
status = pcb_save->sv_env.en_sw;
|
||||||
} else {
|
} else {
|
||||||
@ -567,7 +567,7 @@ fputrap_sse(void)
|
|||||||
|
|
||||||
critical_enter();
|
critical_enter();
|
||||||
if (PCPU_GET(fpcurthread) != curthread)
|
if (PCPU_GET(fpcurthread) != curthread)
|
||||||
mxcsr = PCPU_GET(curpcb)->pcb_save->sv_env.en_mxcsr;
|
mxcsr = curpcb->pcb_save->sv_env.en_mxcsr;
|
||||||
else
|
else
|
||||||
stmxcsr(&mxcsr);
|
stmxcsr(&mxcsr);
|
||||||
critical_exit();
|
critical_exit();
|
||||||
@ -609,7 +609,7 @@ fpudna(void)
|
|||||||
* Record new context early in case frstor causes a trap.
|
* Record new context early in case frstor causes a trap.
|
||||||
*/
|
*/
|
||||||
PCPU_SET(fpcurthread, curthread);
|
PCPU_SET(fpcurthread, curthread);
|
||||||
pcb = PCPU_GET(curpcb);
|
pcb = curpcb;
|
||||||
|
|
||||||
fpu_clean_state();
|
fpu_clean_state();
|
||||||
|
|
||||||
@ -970,7 +970,7 @@ fpu_kern_thread(u_int flags)
|
|||||||
{
|
{
|
||||||
struct pcb *pcb;
|
struct pcb *pcb;
|
||||||
|
|
||||||
pcb = PCPU_GET(curpcb);
|
pcb = curpcb;
|
||||||
KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
|
KASSERT((curthread->td_pflags & TDP_KTHREAD) != 0,
|
||||||
("Only kthread may use fpu_kern_thread"));
|
("Only kthread may use fpu_kern_thread"));
|
||||||
KASSERT(pcb->pcb_save == get_pcb_user_save_pcb(pcb),
|
KASSERT(pcb->pcb_save == get_pcb_user_save_pcb(pcb),
|
||||||
@ -987,5 +987,5 @@ is_fpu_kern_thread(u_int flags)
|
|||||||
|
|
||||||
if ((curthread->td_pflags & TDP_KTHREAD) == 0)
|
if ((curthread->td_pflags & TDP_KTHREAD) == 0)
|
||||||
return (0);
|
return (0);
|
||||||
return ((PCPU_GET(curpcb)->pcb_flags & PCB_KERNFPU) != 0);
|
return ((curpcb->pcb_flags & PCB_KERNFPU) != 0);
|
||||||
}
|
}
|
||||||
|
@ -996,7 +996,7 @@ exec_setregs(struct thread *td, struct image_params *imgp, u_long stack)
|
|||||||
pcb->pcb_dr3 = 0;
|
pcb->pcb_dr3 = 0;
|
||||||
pcb->pcb_dr6 = 0;
|
pcb->pcb_dr6 = 0;
|
||||||
pcb->pcb_dr7 = 0;
|
pcb->pcb_dr7 = 0;
|
||||||
if (pcb == PCPU_GET(curpcb)) {
|
if (pcb == curpcb) {
|
||||||
/*
|
/*
|
||||||
* Clear the debug registers on the running
|
* Clear the debug registers on the running
|
||||||
* CPU, otherwise they will end up affecting
|
* CPU, otherwise they will end up affecting
|
||||||
|
@ -520,9 +520,8 @@ trap(struct trapframe *frame)
|
|||||||
frame->tf_rip = (long)fsbase_load_fault;
|
frame->tf_rip = (long)fsbase_load_fault;
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
if (PCPU_GET(curpcb)->pcb_onfault != NULL) {
|
if (curpcb->pcb_onfault != NULL) {
|
||||||
frame->tf_rip =
|
frame->tf_rip = (long)curpcb->pcb_onfault;
|
||||||
(long)PCPU_GET(curpcb)->pcb_onfault;
|
|
||||||
goto out;
|
goto out;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
@ -708,7 +707,7 @@ trap_pfault(frame, usermode)
|
|||||||
* it normally, and panic immediately.
|
* it normally, and panic immediately.
|
||||||
*/
|
*/
|
||||||
if (!usermode && (td->td_intr_nesting_level != 0 ||
|
if (!usermode && (td->td_intr_nesting_level != 0 ||
|
||||||
PCPU_GET(curpcb)->pcb_onfault == NULL)) {
|
curpcb->pcb_onfault == NULL)) {
|
||||||
trap_fatal(frame, eva);
|
trap_fatal(frame, eva);
|
||||||
return (-1);
|
return (-1);
|
||||||
}
|
}
|
||||||
@ -764,8 +763,8 @@ trap_pfault(frame, usermode)
|
|||||||
nogo:
|
nogo:
|
||||||
if (!usermode) {
|
if (!usermode) {
|
||||||
if (td->td_intr_nesting_level == 0 &&
|
if (td->td_intr_nesting_level == 0 &&
|
||||||
PCPU_GET(curpcb)->pcb_onfault != NULL) {
|
curpcb->pcb_onfault != NULL) {
|
||||||
frame->tf_rip = (long)PCPU_GET(curpcb)->pcb_onfault;
|
frame->tf_rip = (long)curpcb->pcb_onfault;
|
||||||
return (0);
|
return (0);
|
||||||
}
|
}
|
||||||
trap_fatal(frame, eva);
|
trap_fatal(frame, eva);
|
||||||
|
@ -90,6 +90,10 @@ static u_int cpu_reset_proxyid;
|
|||||||
static volatile u_int cpu_reset_proxy_active;
|
static volatile u_int cpu_reset_proxy_active;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
CTASSERT((struct thread **)OFFSETOF_CURTHREAD ==
|
||||||
|
&((struct pcpu *)NULL)->pc_curthread);
|
||||||
|
CTASSERT((struct pcb **)OFFSETOF_CURPCB == &((struct pcpu *)NULL)->pc_curpcb);
|
||||||
|
|
||||||
struct savefpu *
|
struct savefpu *
|
||||||
get_pcb_user_save_td(struct thread *td)
|
get_pcb_user_save_td(struct thread *td)
|
||||||
{
|
{
|
||||||
|
@ -216,16 +216,29 @@ extern struct pcpu *pcpup;
|
|||||||
#define PCPU_PTR(member) __PCPU_PTR(pc_ ## member)
|
#define PCPU_PTR(member) __PCPU_PTR(pc_ ## member)
|
||||||
#define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val)
|
#define PCPU_SET(member, val) __PCPU_SET(pc_ ## member, val)
|
||||||
|
|
||||||
|
#define OFFSETOF_CURTHREAD 0
|
||||||
static __inline __pure2 struct thread *
|
static __inline __pure2 struct thread *
|
||||||
__curthread(void)
|
__curthread(void)
|
||||||
{
|
{
|
||||||
struct thread *td;
|
struct thread *td;
|
||||||
|
|
||||||
__asm("movq %%gs:0,%0" : "=r" (td));
|
__asm("movq %%gs:%1,%0" : "=r" (td)
|
||||||
|
: "m" (*(char *)OFFSETOF_CURTHREAD));
|
||||||
return (td);
|
return (td);
|
||||||
}
|
}
|
||||||
#define curthread (__curthread())
|
#define curthread (__curthread())
|
||||||
|
|
||||||
|
#define OFFSETOF_CURPCB 32
|
||||||
|
static __inline __pure2 struct pcb *
|
||||||
|
__curpcb(void)
|
||||||
|
{
|
||||||
|
struct pcb *pcb;
|
||||||
|
|
||||||
|
__asm("movq %%gs:%1,%0" : "=r" (pcb) : "m" (*(char *)OFFSETOF_CURPCB));
|
||||||
|
return (pcb);
|
||||||
|
}
|
||||||
|
#define curpcb (__curpcb())
|
||||||
|
|
||||||
#define IS_BSP() (PCPU_GET(cpuid) == 0)
|
#define IS_BSP() (PCPU_GET(cpuid) == 0)
|
||||||
|
|
||||||
#else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */
|
#else /* !lint || defined(__GNUCLIKE_ASM) && defined(__GNUCLIKE___TYPEOF) */
|
||||||
|
Loading…
Reference in New Issue
Block a user