1994-08-02 07:55:43 +00:00
|
|
|
/*-
|
2017-11-20 19:43:44 +00:00
|
|
|
* SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
*
|
2003-11-08 04:39:22 +00:00
|
|
|
* Copyright (c) 2003 Peter Wemm.
|
1994-08-02 07:55:43 +00:00
|
|
|
* Copyright (c) 1993 The Regents of the University of California.
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
2017-02-28 23:42:47 +00:00
|
|
|
* 3. Neither the name of the University nor the names of its contributors
|
1994-08-02 07:55:43 +00:00
|
|
|
* may be used to endorse or promote products derived from this software
|
|
|
|
* without specific prior written permission.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
1999-08-28 01:08:13 +00:00
|
|
|
* $FreeBSD$
|
1994-08-02 07:55:43 +00:00
|
|
|
*/
|
|
|
|
|
1993-06-12 14:58:17 +00:00
|
|
|
/*
|
|
|
|
* Functions to provide access to special i386 instructions.
|
2002-02-28 18:26:30 +00:00
|
|
|
* This in included in sys/systm.h, and that file should be
|
|
|
|
* used in preference to this.
|
1993-06-12 14:58:17 +00:00
|
|
|
*/
|
|
|
|
|
1993-11-07 17:43:17 +00:00
|
|
|
#ifndef _MACHINE_CPUFUNC_H_
|
1994-11-14 15:04:06 +00:00
|
|
|
#define _MACHINE_CPUFUNC_H_
|
1993-11-07 17:43:17 +00:00
|
|
|
|
2005-03-02 21:33:29 +00:00
|
|
|
#ifndef _SYS_CDEFS_H_
|
|
|
|
#error this file needs sys/cdefs.h as a prerequisite
|
|
|
|
#endif
|
|
|
|
|
2002-09-22 04:45:21 +00:00
|
|
|
struct region_descriptor;
|
2002-03-27 05:39:23 +00:00
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
#define readb(va) (*(volatile uint8_t *) (va))
|
|
|
|
#define readw(va) (*(volatile uint16_t *) (va))
|
|
|
|
#define readl(va) (*(volatile uint32_t *) (va))
|
|
|
|
#define readq(va) (*(volatile uint64_t *) (va))
|
1998-08-17 08:57:05 +00:00
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
#define writeb(va, d) (*(volatile uint8_t *) (va) = (d))
|
|
|
|
#define writew(va, d) (*(volatile uint16_t *) (va) = (d))
|
|
|
|
#define writel(va, d) (*(volatile uint32_t *) (va) = (d))
|
|
|
|
#define writeq(va, d) (*(volatile uint64_t *) (va) = (d))
|
1998-08-17 08:57:05 +00:00
|
|
|
|
2005-03-02 21:33:29 +00:00
|
|
|
#if defined(__GNUCLIKE_ASM) && defined(__CC_SUPPORTS___INLINE)
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1996-04-07 18:30:56 +00:00
|
|
|
static __inline void
|
|
|
|
breakpoint(void)
|
1994-09-25 21:31:55 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("int $3");
|
1994-09-25 21:31:55 +00:00
|
|
|
}
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 u_int
|
2000-01-09 16:46:03 +00:00
|
|
|
bsfl(u_int mask)
|
|
|
|
{
|
|
|
|
u_int result;
|
|
|
|
|
2001-12-18 08:54:39 +00:00
|
|
|
__asm __volatile("bsfl %1,%0" : "=r" (result) : "rm" (mask));
|
2000-01-09 16:46:03 +00:00
|
|
|
return (result);
|
|
|
|
}
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 u_long
|
2003-12-06 23:22:43 +00:00
|
|
|
bsfq(u_long mask)
|
|
|
|
{
|
|
|
|
u_long result;
|
|
|
|
|
|
|
|
__asm __volatile("bsfq %1,%0" : "=r" (result) : "rm" (mask));
|
|
|
|
return (result);
|
|
|
|
}
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 u_int
|
2000-01-09 16:46:03 +00:00
|
|
|
bsrl(u_int mask)
|
|
|
|
{
|
|
|
|
u_int result;
|
|
|
|
|
2001-12-18 08:54:39 +00:00
|
|
|
__asm __volatile("bsrl %1,%0" : "=r" (result) : "rm" (mask));
|
2000-01-09 16:46:03 +00:00
|
|
|
return (result);
|
|
|
|
}
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 u_long
|
2003-12-06 23:22:43 +00:00
|
|
|
bsrq(u_long mask)
|
|
|
|
{
|
|
|
|
u_long result;
|
|
|
|
|
|
|
|
__asm __volatile("bsrq %1,%0" : "=r" (result) : "rm" (mask));
|
|
|
|
return (result);
|
|
|
|
}
|
|
|
|
|
2009-07-22 14:32:38 +00:00
|
|
|
static __inline void
|
|
|
|
clflush(u_long addr)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("clflush %0" : : "m" (*(char *)addr));
|
|
|
|
}
|
|
|
|
|
2015-10-23 11:45:38 +00:00
|
|
|
static __inline void
|
|
|
|
clflushopt(u_long addr)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile(".byte 0x66;clflush %0" : : "m" (*(char *)addr));
|
|
|
|
}
|
|
|
|
|
2018-10-16 17:00:42 +00:00
|
|
|
static __inline void
|
|
|
|
clwb(u_long addr)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("clwb %0" : : "m" (*(char *)addr));
|
|
|
|
}
|
|
|
|
|
2012-07-09 20:55:39 +00:00
|
|
|
static __inline void
|
|
|
|
clts(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("clts");
|
|
|
|
}
|
|
|
|
|
1994-09-25 21:31:55 +00:00
|
|
|
static __inline void
|
1994-11-14 15:04:06 +00:00
|
|
|
disable_intr(void)
|
1994-09-25 21:31:55 +00:00
|
|
|
{
|
1995-08-08 04:50:52 +00:00
|
|
|
__asm __volatile("cli" : : : "memory");
|
1994-09-25 21:31:55 +00:00
|
|
|
}
|
|
|
|
|
2002-04-10 21:18:46 +00:00
|
|
|
static __inline void
|
|
|
|
do_cpuid(u_int ax, u_int *p)
|
|
|
|
{
|
|
|
|
__asm __volatile("cpuid"
|
|
|
|
: "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
|
|
|
|
: "0" (ax));
|
|
|
|
}
|
|
|
|
|
2005-05-13 00:10:56 +00:00
|
|
|
static __inline void
|
|
|
|
cpuid_count(u_int ax, u_int cx, u_int *p)
|
|
|
|
{
|
|
|
|
__asm __volatile("cpuid"
|
|
|
|
: "=a" (p[0]), "=b" (p[1]), "=c" (p[2]), "=d" (p[3])
|
|
|
|
: "0" (ax), "c" (cx));
|
|
|
|
}
|
|
|
|
|
1994-09-25 21:31:55 +00:00
|
|
|
static __inline void
|
1994-11-14 15:04:06 +00:00
|
|
|
enable_intr(void)
|
1994-09-25 21:31:55 +00:00
|
|
|
{
|
1995-08-26 20:45:59 +00:00
|
|
|
__asm __volatile("sti");
|
1994-09-25 21:31:55 +00:00
|
|
|
}
|
|
|
|
|
2004-03-11 13:38:54 +00:00
|
|
|
#ifdef _KERNEL
|
|
|
|
|
1999-08-19 14:54:40 +00:00
|
|
|
#define HAVE_INLINE_FFS
|
2004-07-30 16:44:29 +00:00
|
|
|
#define ffs(x) __builtin_ffs(x)
|
2003-12-06 23:22:43 +00:00
|
|
|
|
|
|
|
#define HAVE_INLINE_FFSL
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 int
|
2003-12-06 23:22:43 +00:00
|
|
|
ffsl(long mask)
|
|
|
|
{
|
|
|
|
return (mask == 0 ? mask : (int)bsfq((u_long)mask) + 1);
|
1994-09-25 21:31:55 +00:00
|
|
|
}
|
|
|
|
|
2014-02-14 15:18:37 +00:00
|
|
|
#define HAVE_INLINE_FFSLL
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 int
|
2014-02-14 15:18:37 +00:00
|
|
|
ffsll(long long mask)
|
|
|
|
{
|
|
|
|
return (ffsl((long)mask));
|
|
|
|
}
|
|
|
|
|
1996-08-01 20:29:28 +00:00
|
|
|
#define HAVE_INLINE_FLS
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 int
|
1996-08-01 20:29:28 +00:00
|
|
|
fls(int mask)
|
|
|
|
{
|
2002-07-15 13:27:43 +00:00
|
|
|
return (mask == 0 ? mask : (int)bsrl((u_int)mask) + 1);
|
1996-08-01 20:29:28 +00:00
|
|
|
}
|
|
|
|
|
2003-12-06 23:22:43 +00:00
|
|
|
#define HAVE_INLINE_FLSL
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 int
|
2003-12-06 23:22:43 +00:00
|
|
|
flsl(long mask)
|
|
|
|
{
|
|
|
|
return (mask == 0 ? mask : (int)bsrq((u_long)mask) + 1);
|
|
|
|
}
|
|
|
|
|
2014-02-14 15:18:37 +00:00
|
|
|
#define HAVE_INLINE_FLSLL
|
|
|
|
|
2017-08-03 22:28:30 +00:00
|
|
|
static __inline __pure2 int
|
2014-02-14 15:18:37 +00:00
|
|
|
flsll(long long mask)
|
|
|
|
{
|
|
|
|
return (flsl((long)mask));
|
|
|
|
}
|
|
|
|
|
2004-03-11 13:38:54 +00:00
|
|
|
#endif /* _KERNEL */
|
|
|
|
|
2002-09-21 18:26:53 +00:00
|
|
|
static __inline void
|
|
|
|
halt(void)
|
|
|
|
{
|
|
|
|
__asm __volatile("hlt");
|
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline u_char
|
2009-04-11 14:01:01 +00:00
|
|
|
inb(u_int port)
|
1994-09-25 21:31:55 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
u_char data;
|
|
|
|
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("inb %w1, %0" : "=a" (data) : "Nd" (port));
|
1994-11-14 15:04:06 +00:00
|
|
|
return (data);
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1998-07-11 04:58:25 +00:00
|
|
|
static __inline u_int
|
1994-11-14 15:04:06 +00:00
|
|
|
inl(u_int port)
|
1994-09-16 13:33:56 +00:00
|
|
|
{
|
1998-07-11 04:58:25 +00:00
|
|
|
u_int data;
|
1994-09-16 13:33:56 +00:00
|
|
|
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("inl %w1, %0" : "=a" (data) : "Nd" (port));
|
1994-11-14 15:04:06 +00:00
|
|
|
return (data);
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
insb(u_int port, void *addr, size_t count)
|
1994-09-16 13:33:56 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; insb"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+D" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port)
|
1999-01-09 13:00:27 +00:00
|
|
|
: "memory");
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
insw(u_int port, void *addr, size_t count)
|
1994-09-16 13:33:56 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; insw"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+D" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port)
|
1999-01-09 13:00:27 +00:00
|
|
|
: "memory");
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
insl(u_int port, void *addr, size_t count)
|
1994-09-16 13:33:56 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; insl"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+D" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port)
|
1999-01-09 13:00:27 +00:00
|
|
|
: "memory");
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1997-03-22 18:54:54 +00:00
|
|
|
static __inline void
|
|
|
|
invd(void)
|
|
|
|
{
|
|
|
|
__asm __volatile("invd");
|
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline u_short
|
|
|
|
inw(u_int port)
|
1994-09-16 13:33:56 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
u_short data;
|
|
|
|
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("inw %w1, %0" : "=a" (data) : "Nd" (port));
|
1994-11-14 15:04:06 +00:00
|
|
|
return (data);
|
1994-09-16 13:33:56 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2009-04-11 14:01:01 +00:00
|
|
|
outb(u_int port, u_char data)
|
1993-06-12 14:58:17 +00:00
|
|
|
{
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("outb %0, %w1" : : "a" (data), "Nd" (port));
|
1994-11-14 15:04:06 +00:00
|
|
|
}
|
1993-06-12 14:58:17 +00:00
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
1998-07-11 04:58:25 +00:00
|
|
|
outl(u_int port, u_int data)
|
1994-11-14 15:04:06 +00:00
|
|
|
{
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("outl %0, %w1" : : "a" (data), "Nd" (port));
|
1993-06-12 14:58:17 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
outsb(u_int port, const void *addr, size_t count)
|
1993-06-12 14:58:17 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; outsb"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+S" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port));
|
1993-06-12 14:58:17 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
outsw(u_int port, const void *addr, size_t count)
|
1993-06-12 14:58:17 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; outsw"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+S" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port));
|
1993-06-12 14:58:17 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
2010-01-01 20:55:11 +00:00
|
|
|
outsl(u_int port, const void *addr, size_t count)
|
1993-06-12 14:58:17 +00:00
|
|
|
{
|
1994-11-14 15:04:06 +00:00
|
|
|
__asm __volatile("cld; rep; outsl"
|
2010-01-01 20:55:11 +00:00
|
|
|
: "+S" (addr), "+c" (count)
|
2001-12-18 08:54:39 +00:00
|
|
|
: "d" (port));
|
1993-06-12 14:58:17 +00:00
|
|
|
}
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
static __inline void
|
|
|
|
outw(u_int port, u_short data)
|
1993-06-12 14:58:17 +00:00
|
|
|
{
|
2011-04-14 16:19:41 +00:00
|
|
|
__asm __volatile("outw %0, %w1" : : "a" (data), "Nd" (port));
|
1993-06-12 14:58:17 +00:00
|
|
|
}
|
|
|
|
|
2012-06-30 20:25:12 +00:00
|
|
|
static __inline u_long
|
|
|
|
popcntq(u_long mask)
|
|
|
|
{
|
|
|
|
u_long result;
|
|
|
|
|
|
|
|
__asm __volatile("popcntq %1,%0" : "=r" (result) : "rm" (mask));
|
|
|
|
return (result);
|
|
|
|
}
|
|
|
|
|
2012-08-01 17:24:53 +00:00
|
|
|
static __inline void
|
|
|
|
lfence(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("lfence" : : : "memory");
|
|
|
|
}
|
|
|
|
|
2009-07-22 14:32:38 +00:00
|
|
|
static __inline void
|
|
|
|
mfence(void)
|
|
|
|
{
|
|
|
|
|
2009-09-30 16:34:50 +00:00
|
|
|
__asm __volatile("mfence" : : : "memory");
|
2009-07-22 14:32:38 +00:00
|
|
|
}
|
|
|
|
|
2017-01-20 19:08:44 +00:00
|
|
|
static __inline void
|
|
|
|
sfence(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("sfence" : : : "memory");
|
|
|
|
}
|
|
|
|
|
2002-05-22 13:27:05 +00:00
|
|
|
static __inline void
|
2002-05-22 20:32:39 +00:00
|
|
|
ia32_pause(void)
|
2002-05-22 13:27:05 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("pause");
|
|
|
|
}
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
static __inline u_long
|
|
|
|
read_rflags(void)
|
1994-11-14 15:04:06 +00:00
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
u_long rf;
|
1994-11-14 15:04:06 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("pushfq; popq %0" : "=r" (rf));
|
|
|
|
return (rf);
|
1994-11-14 15:04:06 +00:00
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
1996-04-07 18:30:56 +00:00
|
|
|
rdmsr(u_int msr)
|
1994-11-14 15:04:06 +00:00
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint32_t low, high;
|
1996-04-07 18:30:56 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("rdmsr" : "=a" (low), "=d" (high) : "c" (msr));
|
2011-04-14 16:14:35 +00:00
|
|
|
return (low | ((uint64_t)high << 32));
|
1994-06-06 14:54:41 +00:00
|
|
|
}
|
|
|
|
|
2015-02-09 21:00:56 +00:00
|
|
|
static __inline uint32_t
|
|
|
|
rdmsr32(u_int msr)
|
|
|
|
{
|
|
|
|
uint32_t low;
|
|
|
|
|
|
|
|
__asm __volatile("rdmsr" : "=a" (low) : "c" (msr) : "rdx");
|
|
|
|
return (low);
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
1996-04-07 18:30:56 +00:00
|
|
|
rdpmc(u_int pmc)
|
1996-03-26 19:57:56 +00:00
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint32_t low, high;
|
1996-04-07 18:30:56 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("rdpmc" : "=a" (low), "=d" (high) : "c" (pmc));
|
2011-04-14 16:14:35 +00:00
|
|
|
return (low | ((uint64_t)high << 32));
|
1996-03-26 19:57:56 +00:00
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
1996-03-26 19:57:56 +00:00
|
|
|
rdtsc(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint32_t low, high;
|
1996-04-07 18:30:56 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("rdtsc" : "=a" (low), "=d" (high));
|
2011-04-14 16:14:35 +00:00
|
|
|
return (low | ((uint64_t)high << 32));
|
1996-03-26 19:57:56 +00:00
|
|
|
}
|
|
|
|
|
2018-06-07 00:54:11 +00:00
|
|
|
static __inline uint64_t
|
|
|
|
rdtscp(void)
|
|
|
|
{
|
|
|
|
uint32_t low, high;
|
|
|
|
|
2018-06-09 18:31:19 +00:00
|
|
|
__asm __volatile("rdtscp" : "=a" (low), "=d" (high) : : "ecx");
|
2018-06-07 00:54:11 +00:00
|
|
|
return (low | ((uint64_t)high << 32));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:53:32 +00:00
|
|
|
static __inline uint32_t
|
|
|
|
rdtsc32(void)
|
|
|
|
{
|
|
|
|
uint32_t rv;
|
|
|
|
|
|
|
|
__asm __volatile("rdtsc" : "=a" (rv) : : "edx");
|
|
|
|
return (rv);
|
|
|
|
}
|
|
|
|
|
1997-03-22 18:54:54 +00:00
|
|
|
static __inline void
|
|
|
|
wbinvd(void)
|
|
|
|
{
|
|
|
|
__asm __volatile("wbinvd");
|
|
|
|
}
|
|
|
|
|
1996-04-07 18:30:56 +00:00
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
write_rflags(u_long rf)
|
1996-03-26 19:57:56 +00:00
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("pushq %0; popfq" : : "r" (rf));
|
1996-03-26 19:57:56 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
wrmsr(u_int msr, uint64_t newval)
|
1996-03-26 19:57:56 +00:00
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint32_t low, high;
|
2003-05-01 01:05:25 +00:00
|
|
|
|
|
|
|
low = newval;
|
|
|
|
high = newval >> 32;
|
|
|
|
__asm __volatile("wrmsr" : : "a" (low), "d" (high), "c" (msr));
|
1996-03-26 19:57:56 +00:00
|
|
|
}
|
|
|
|
|
2002-07-12 07:56:11 +00:00
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
load_cr0(u_long data)
|
2002-07-12 07:56:11 +00:00
|
|
|
{
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %0,%%cr0" : : "r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
}
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
static __inline u_long
|
2002-07-12 07:56:11 +00:00
|
|
|
rcr0(void)
|
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
u_long data;
|
2002-07-12 07:56:11 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %%cr0,%0" : "=r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
static __inline u_long
|
2002-07-12 07:56:11 +00:00
|
|
|
rcr2(void)
|
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
u_long data;
|
2002-07-12 07:56:11 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %%cr2,%0" : "=r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
load_cr3(u_long data)
|
2002-07-12 07:56:11 +00:00
|
|
|
{
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %0,%%cr3" : : "r" (data) : "memory");
|
2002-07-12 07:56:11 +00:00
|
|
|
}
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
static __inline u_long
|
2002-07-12 07:56:11 +00:00
|
|
|
rcr3(void)
|
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
u_long data;
|
2002-07-12 07:56:11 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %%cr3,%0" : "=r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
load_cr4(u_long data)
|
2002-07-12 07:56:11 +00:00
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %0,%%cr4" : : "r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
}
|
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
static __inline u_long
|
2002-07-12 07:56:11 +00:00
|
|
|
rcr4(void)
|
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
u_long data;
|
2002-07-12 07:56:11 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
__asm __volatile("movq %%cr4,%0" : "=r" (data));
|
2002-07-12 07:56:11 +00:00
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
2012-07-05 18:19:35 +00:00
|
|
|
static __inline u_long
|
|
|
|
rxcr(u_int reg)
|
|
|
|
{
|
|
|
|
u_int low, high;
|
|
|
|
|
|
|
|
__asm __volatile("xgetbv" : "=a" (low), "=d" (high) : "c" (reg));
|
|
|
|
return (low | ((uint64_t)high << 32));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
load_xcr(u_int reg, u_long val)
|
|
|
|
{
|
|
|
|
u_int low, high;
|
|
|
|
|
|
|
|
low = val;
|
|
|
|
high = val >> 32;
|
|
|
|
__asm __volatile("xsetbv" : : "c" (reg), "a" (low), "d" (high));
|
|
|
|
}
|
|
|
|
|
2002-07-12 07:56:11 +00:00
|
|
|
/*
|
|
|
|
* Global TLB flush (except for thise for pages marked PG_G)
|
|
|
|
*/
|
|
|
|
static __inline void
|
|
|
|
invltlb(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
load_cr3(rcr3());
|
|
|
|
}
|
|
|
|
|
2013-09-06 22:17:02 +00:00
|
|
|
#ifndef CR4_PGE
|
|
|
|
#define CR4_PGE 0x00000080 /* Page global enable */
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Perform the guaranteed invalidation of all TLB entries. This
|
|
|
|
* includes the global entries, and entries in all PCIDs, not only the
|
|
|
|
* current context. The function works both on non-PCID CPUs and CPUs
|
|
|
|
* with the PCID turned off or on. See IA-32 SDM Vol. 3a 4.10.4.1
|
|
|
|
* Operations that Invalidate TLBs and Paging-Structure Caches.
|
|
|
|
*/
|
|
|
|
static __inline void
|
For amd64 non-PCID machines, and for i386 machines with support for
the PG_G global pte flag, pmap_invalidate_all() fails to flush global
TLB entries [*]. This is because TLB shootdown handler for such
configs reloads CR3, and on i386 pmap_invalidate_all() does the same
for the initiating CPU. Note that current code does not issue total
invalidation requests for the kernel_pmap.
Rename amd64 function invltlb_globpcid() to invltlb_glob(), it is not
specific for PCID for quite some time, and implement the same
functionality for i386. Use the function instead of invltlb() in
shootdown handlers and in i386 pmap_invalidate_all(), but only for the
kernel pmap (which maps pages with the PG_G attribute set), which
takes care of PG_G TLB entries on flush.
To detect the affected pmap in i386 TLB shootdown handler, pmap should
be passed to the smp_masked_invltlb() function, which makes amd64 and
i386 TLB shootdown code almost identical. Merge the code under x86/.
Noted by: jhb [*]
Reviewed by: cem, jhb, pho
Tested by: pho
Sponsored by: The FreeBSD Foundation
Differential revision: https://reviews.freebsd.org/D4346
2015-12-03 11:14:14 +00:00
|
|
|
invltlb_glob(void)
|
2013-09-06 22:17:02 +00:00
|
|
|
{
|
|
|
|
uint64_t cr4;
|
|
|
|
|
|
|
|
cr4 = rcr4();
|
|
|
|
load_cr4(cr4 & ~CR4_PGE);
|
|
|
|
/*
|
|
|
|
* Although preemption at this point could be detrimental to
|
|
|
|
* performance, it would not lead to an error. PG_G is simply
|
|
|
|
* ignored if CR4.PGE is clear. Moreover, in case this block
|
|
|
|
* is re-entered, the load_cr4() either above or below will
|
|
|
|
* modify CR4.PGE flushing the TLB.
|
|
|
|
*/
|
|
|
|
load_cr4(cr4 | CR4_PGE);
|
|
|
|
}
|
|
|
|
|
2002-07-12 07:56:11 +00:00
|
|
|
/*
|
|
|
|
* TLB flush for an individual page (even if it has PG_G).
|
|
|
|
* Only works on 486+ CPUs (i386 does not have PG_G).
|
|
|
|
*/
|
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
invlpg(u_long addr)
|
2002-07-12 07:56:11 +00:00
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
|
|
|
|
}
|
|
|
|
|
2013-08-30 07:42:38 +00:00
|
|
|
#define INVPCID_ADDR 0
|
|
|
|
#define INVPCID_CTX 1
|
|
|
|
#define INVPCID_CTXGLOB 2
|
|
|
|
#define INVPCID_ALLCTX 3
|
|
|
|
|
|
|
|
struct invpcid_descr {
|
|
|
|
uint64_t pcid:12 __packed;
|
|
|
|
uint64_t pad:52 __packed;
|
|
|
|
uint64_t addr;
|
|
|
|
} __packed;
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
invpcid(struct invpcid_descr *d, int type)
|
|
|
|
{
|
|
|
|
|
Rewrite amd64 PCID implementation to follow an algorithm described in
the Vahalia' "Unix Internals" section 15.12 "Other TLB Consistency
Algorithms". The same algorithm is already utilized by the MIPS pmap
to handle ASIDs.
The PCID for the address space is now allocated per-cpu during context
switch to the thread using pmap, when no PCID on the cpu was ever
allocated, or the current PCID is invalidated. If the PCID is reused,
bit 63 of %cr3 can be set to avoid TLB flush.
Each cpu has PCID' algorithm generation count, which is saved in the
pmap pcpu block when pcpu PCID is allocated. On invalidation, the
pmap generation count is zeroed, which signals the context switch code
that already allocated PCID is no longer valid. The implication is
the TLB shootdown for the given cpu/address space, due to the
allocation of new PCID.
The pm_save mask is no longer has to be tracked, which (significantly)
reduces the targets of the TLB shootdown IPIs. Previously, pm_save
was reset only on pmap_invalidate_all(), which made it accumulate the
cpuids of all processors on which the thread was scheduled between
full TLB shootdowns.
Besides reducing the amount of TLB shootdowns and removing atomics to
update pm_saves in the context switch code, the algorithm is much
simpler than the maintanence of pm_save and selection of the right
address space in the shootdown IPI handler.
Reviewed by: alc
Tested by: pho
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-05-09 19:11:01 +00:00
|
|
|
__asm __volatile("invpcid (%0),%1"
|
|
|
|
: : "r" (d), "r" ((u_long)type) : "memory");
|
2013-08-30 07:42:38 +00:00
|
|
|
}
|
|
|
|
|
2010-09-03 14:25:17 +00:00
|
|
|
static __inline u_short
|
1999-04-28 01:04:33 +00:00
|
|
|
rfs(void)
|
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
u_short sel;
|
|
|
|
__asm __volatile("movw %%fs,%0" : "=rm" (sel));
|
1999-04-28 01:04:33 +00:00
|
|
|
return (sel);
|
|
|
|
}
|
|
|
|
|
2010-09-03 14:25:17 +00:00
|
|
|
static __inline u_short
|
1999-04-28 01:04:33 +00:00
|
|
|
rgs(void)
|
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
u_short sel;
|
|
|
|
__asm __volatile("movw %%gs,%0" : "=rm" (sel));
|
1999-04-28 01:04:33 +00:00
|
|
|
return (sel);
|
|
|
|
}
|
|
|
|
|
2010-09-03 14:25:17 +00:00
|
|
|
static __inline u_short
|
2004-04-07 00:41:05 +00:00
|
|
|
rss(void)
|
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
u_short sel;
|
|
|
|
__asm __volatile("movw %%ss,%0" : "=rm" (sel));
|
2004-04-07 00:41:05 +00:00
|
|
|
return (sel);
|
|
|
|
}
|
|
|
|
|
2003-05-14 04:10:49 +00:00
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_ds(u_short sel)
|
2003-05-14 04:10:49 +00:00
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("movw %0,%%ds" : : "rm" (sel));
|
2003-05-14 04:10:49 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_es(u_short sel)
|
2003-05-14 04:10:49 +00:00
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("movw %0,%%es" : : "rm" (sel));
|
2003-05-14 04:10:49 +00:00
|
|
|
}
|
|
|
|
|
2009-09-30 16:34:50 +00:00
|
|
|
static __inline void
|
2011-07-05 18:42:10 +00:00
|
|
|
cpu_monitor(const void *addr, u_long extensions, u_int hints)
|
2008-04-18 05:47:56 +00:00
|
|
|
{
|
2011-07-05 18:42:10 +00:00
|
|
|
|
|
|
|
__asm __volatile("monitor"
|
|
|
|
: : "a" (addr), "c" (extensions), "d" (hints));
|
2008-04-18 05:47:56 +00:00
|
|
|
}
|
|
|
|
|
2009-09-30 16:34:50 +00:00
|
|
|
static __inline void
|
2011-07-05 18:42:10 +00:00
|
|
|
cpu_mwait(u_long extensions, u_int hints)
|
2008-04-18 05:47:56 +00:00
|
|
|
{
|
2011-07-05 18:42:10 +00:00
|
|
|
|
|
|
|
__asm __volatile("mwait" : : "a" (hints), "c" (extensions));
|
2008-04-18 05:47:56 +00:00
|
|
|
}
|
|
|
|
|
2019-02-19 19:17:20 +00:00
|
|
|
static __inline uint32_t
|
|
|
|
rdpkru(void)
|
|
|
|
{
|
|
|
|
uint32_t res;
|
|
|
|
|
|
|
|
__asm __volatile("rdpkru" : "=a" (res) : "c" (0) : "edx");
|
|
|
|
return (res);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
wrpkru(uint32_t mask)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("wrpkru" : : "a" (mask), "c" (0), "d" (0));
|
|
|
|
}
|
|
|
|
|
2003-05-15 00:23:40 +00:00
|
|
|
#ifdef _KERNEL
|
|
|
|
/* This is defined in <machine/specialreg.h> but is too painful to get to */
|
|
|
|
#ifndef MSR_FSBASE
|
|
|
|
#define MSR_FSBASE 0xc0000100
|
|
|
|
#endif
|
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_fs(u_short sel)
|
2003-05-15 00:23:40 +00:00
|
|
|
{
|
|
|
|
/* Preserve the fsbase value across the selector load */
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("rdmsr; movw %0,%%fs; wrmsr"
|
2009-04-07 19:31:36 +00:00
|
|
|
: : "rm" (sel), "c" (MSR_FSBASE) : "eax", "edx");
|
2003-05-15 00:23:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifndef MSR_GSBASE
|
|
|
|
#define MSR_GSBASE 0xc0000101
|
|
|
|
#endif
|
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_gs(u_short sel)
|
2003-05-15 00:23:40 +00:00
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Preserve the gsbase value across the selector load.
|
|
|
|
* Note that we have to disable interrupts because the gsbase
|
|
|
|
* being trashed happens to be the kernel gsbase at the time.
|
|
|
|
*/
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("pushfq; cli; rdmsr; movw %0,%%gs; wrmsr; popfq"
|
2009-04-07 19:31:36 +00:00
|
|
|
: : "rm" (sel), "c" (MSR_GSBASE) : "eax", "edx");
|
2003-05-15 00:23:40 +00:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
/* Usable by userland */
|
1999-04-28 01:04:33 +00:00
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_fs(u_short sel)
|
1999-04-28 01:04:33 +00:00
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("movw %0,%%fs" : : "rm" (sel));
|
1999-04-28 01:04:33 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2010-09-03 14:25:17 +00:00
|
|
|
load_gs(u_short sel)
|
1999-04-28 01:04:33 +00:00
|
|
|
{
|
2010-09-03 14:25:17 +00:00
|
|
|
__asm __volatile("movw %0,%%gs" : : "rm" (sel));
|
1999-04-28 01:04:33 +00:00
|
|
|
}
|
2003-05-15 00:23:40 +00:00
|
|
|
#endif
|
1999-04-28 01:04:33 +00:00
|
|
|
|
2017-08-14 11:20:54 +00:00
|
|
|
static __inline uint64_t
|
|
|
|
rdfsbase(void)
|
|
|
|
{
|
|
|
|
uint64_t x;
|
|
|
|
|
|
|
|
__asm __volatile("rdfsbase %0" : "=r" (x));
|
|
|
|
return (x);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
wrfsbase(uint64_t x)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("wrfsbase %0" : : "r" (x));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline uint64_t
|
|
|
|
rdgsbase(void)
|
|
|
|
{
|
|
|
|
uint64_t x;
|
|
|
|
|
|
|
|
__asm __volatile("rdgsbase %0" : "=r" (x));
|
|
|
|
return (x);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
wrgsbase(uint64_t x)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("wrgsbase %0" : : "r" (x));
|
|
|
|
}
|
|
|
|
|
2016-09-21 10:10:36 +00:00
|
|
|
static __inline void
|
|
|
|
bare_lgdt(struct region_descriptor *addr)
|
|
|
|
{
|
|
|
|
__asm __volatile("lgdt (%0)" : : "r" (addr));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
sgdt(struct region_descriptor *addr)
|
|
|
|
{
|
|
|
|
char *loc;
|
|
|
|
|
|
|
|
loc = (char *)addr;
|
|
|
|
__asm __volatile("sgdt %0" : "=m" (*loc) : : "memory");
|
|
|
|
}
|
|
|
|
|
2002-09-22 04:45:21 +00:00
|
|
|
static __inline void
|
|
|
|
lidt(struct region_descriptor *addr)
|
|
|
|
{
|
|
|
|
__asm __volatile("lidt (%0)" : : "r" (addr));
|
|
|
|
}
|
|
|
|
|
2016-09-21 10:10:36 +00:00
|
|
|
static __inline void
|
|
|
|
sidt(struct region_descriptor *addr)
|
|
|
|
{
|
|
|
|
char *loc;
|
|
|
|
|
|
|
|
loc = (char *)addr;
|
|
|
|
__asm __volatile("sidt %0" : "=m" (*loc) : : "memory");
|
|
|
|
}
|
|
|
|
|
2002-09-22 04:45:21 +00:00
|
|
|
static __inline void
|
|
|
|
lldt(u_short sel)
|
|
|
|
{
|
|
|
|
__asm __volatile("lldt %0" : : "r" (sel));
|
|
|
|
}
|
|
|
|
|
2018-10-11 18:27:19 +00:00
|
|
|
static __inline u_short
|
|
|
|
sldt(void)
|
|
|
|
{
|
|
|
|
u_short sel;
|
|
|
|
|
|
|
|
__asm __volatile("sldt %0" : "=r" (sel));
|
|
|
|
return (sel);
|
|
|
|
}
|
|
|
|
|
2002-09-22 04:45:21 +00:00
|
|
|
static __inline void
|
|
|
|
ltr(u_short sel)
|
|
|
|
{
|
|
|
|
__asm __volatile("ltr %0" : : "r" (sel));
|
|
|
|
}
|
|
|
|
|
2016-09-21 10:10:36 +00:00
|
|
|
static __inline uint32_t
|
|
|
|
read_tr(void)
|
|
|
|
{
|
|
|
|
u_short sel;
|
|
|
|
|
|
|
|
__asm __volatile("str %0" : "=r" (sel));
|
|
|
|
return (sel);
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr0(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr0,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr0(uint64_t dr0)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr0" : : "r" (dr0));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr1(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr1,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr1(uint64_t dr1)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr1" : : "r" (dr1));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr2(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr2,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr2(uint64_t dr2)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr2" : : "r" (dr2));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr3(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr3,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr3(uint64_t dr3)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr3" : : "r" (dr3));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr6(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr6,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr6(uint64_t dr6)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr6" : : "r" (dr6));
|
|
|
|
}
|
|
|
|
|
2011-04-14 16:14:35 +00:00
|
|
|
static __inline uint64_t
|
2004-01-28 23:53:04 +00:00
|
|
|
rdr7(void)
|
|
|
|
{
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t data;
|
2004-01-28 23:53:04 +00:00
|
|
|
__asm __volatile("movq %%dr7,%0" : "=r" (data));
|
|
|
|
return (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2011-04-14 16:14:35 +00:00
|
|
|
load_dr7(uint64_t dr7)
|
2004-01-28 23:53:04 +00:00
|
|
|
{
|
|
|
|
__asm __volatile("movq %0,%%dr7" : : "r" (dr7));
|
|
|
|
}
|
|
|
|
|
2002-03-21 06:19:08 +00:00
|
|
|
static __inline register_t
|
|
|
|
intr_disable(void)
|
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
register_t rflags;
|
2002-03-21 06:19:08 +00:00
|
|
|
|
2003-05-01 01:05:25 +00:00
|
|
|
rflags = read_rflags();
|
2002-03-21 06:19:08 +00:00
|
|
|
disable_intr();
|
2003-05-01 01:05:25 +00:00
|
|
|
return (rflags);
|
2002-03-21 06:19:08 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
2003-05-01 01:05:25 +00:00
|
|
|
intr_restore(register_t rflags)
|
2002-03-21 06:19:08 +00:00
|
|
|
{
|
2003-05-01 01:05:25 +00:00
|
|
|
write_rflags(rflags);
|
2017-08-16 10:38:06 +00:00
|
|
|
}
|
|
|
|
|
2018-01-14 12:39:50 +00:00
|
|
|
static __inline void
|
|
|
|
stac(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("stac" : : : "cc");
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline void
|
|
|
|
clac(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
__asm __volatile("clac" : : : "cc");
|
|
|
|
}
|
|
|
|
|
2017-08-16 10:38:06 +00:00
|
|
|
enum {
|
|
|
|
SGX_ECREATE = 0x0,
|
|
|
|
SGX_EADD = 0x1,
|
|
|
|
SGX_EINIT = 0x2,
|
|
|
|
SGX_EREMOVE = 0x3,
|
|
|
|
SGX_EDGBRD = 0x4,
|
|
|
|
SGX_EDGBWR = 0x5,
|
|
|
|
SGX_EEXTEND = 0x6,
|
|
|
|
SGX_ELDU = 0x8,
|
|
|
|
SGX_EBLOCK = 0x9,
|
|
|
|
SGX_EPA = 0xA,
|
|
|
|
SGX_EWB = 0xB,
|
|
|
|
SGX_ETRACK = 0xC,
|
|
|
|
};
|
|
|
|
|
|
|
|
enum {
|
|
|
|
SGX_PT_SECS = 0x00,
|
|
|
|
SGX_PT_TCS = 0x01,
|
|
|
|
SGX_PT_REG = 0x02,
|
|
|
|
SGX_PT_VA = 0x03,
|
|
|
|
SGX_PT_TRIM = 0x04,
|
|
|
|
};
|
|
|
|
|
|
|
|
int sgx_encls(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx);
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_ecreate(void *pginfo, void *secs)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_ECREATE, (uint64_t)pginfo,
|
|
|
|
(uint64_t)secs, 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_eadd(void *pginfo, void *epc)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_EADD, (uint64_t)pginfo,
|
|
|
|
(uint64_t)epc, 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_einit(void *sigstruct, void *secs, void *einittoken)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_EINIT, (uint64_t)sigstruct,
|
|
|
|
(uint64_t)secs, (uint64_t)einittoken));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_eextend(void *secs, void *epc)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_EEXTEND, (uint64_t)secs,
|
|
|
|
(uint64_t)epc, 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_epa(void *epc)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_EPA, SGX_PT_VA, (uint64_t)epc, 0));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_eldu(uint64_t rbx, uint64_t rcx,
|
|
|
|
uint64_t rdx)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_ELDU, rbx, rcx, rdx));
|
|
|
|
}
|
|
|
|
|
|
|
|
static __inline int
|
|
|
|
sgx_eremove(void *epc)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (sgx_encls(SGX_EREMOVE, 0, (uint64_t)epc, 0));
|
2002-03-21 06:19:08 +00:00
|
|
|
}
|
|
|
|
|
2005-03-02 21:33:29 +00:00
|
|
|
#else /* !(__GNUCLIKE_ASM && __CC_SUPPORTS___INLINE) */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2002-03-20 19:04:56 +00:00
|
|
|
int breakpoint(void);
|
|
|
|
u_int bsfl(u_int mask);
|
|
|
|
u_int bsrl(u_int mask);
|
2012-07-09 20:55:39 +00:00
|
|
|
void clflush(u_long addr);
|
|
|
|
void clts(void);
|
|
|
|
void cpuid_count(u_int ax, u_int cx, u_int *p);
|
2002-03-20 19:04:56 +00:00
|
|
|
void disable_intr(void);
|
|
|
|
void do_cpuid(u_int ax, u_int *p);
|
|
|
|
void enable_intr(void);
|
2002-09-21 18:26:53 +00:00
|
|
|
void halt(void);
|
2004-03-08 00:24:15 +00:00
|
|
|
void ia32_pause(void);
|
2002-03-20 19:04:56 +00:00
|
|
|
u_char inb(u_int port);
|
|
|
|
u_int inl(u_int port);
|
2010-01-01 20:55:11 +00:00
|
|
|
void insb(u_int port, void *addr, size_t count);
|
|
|
|
void insl(u_int port, void *addr, size_t count);
|
|
|
|
void insw(u_int port, void *addr, size_t count);
|
2004-03-08 00:24:15 +00:00
|
|
|
register_t intr_disable(void);
|
|
|
|
void intr_restore(register_t rf);
|
2002-03-20 19:04:56 +00:00
|
|
|
void invd(void);
|
|
|
|
void invlpg(u_int addr);
|
|
|
|
void invltlb(void);
|
|
|
|
u_short inw(u_int port);
|
2002-09-22 04:45:21 +00:00
|
|
|
void lidt(struct region_descriptor *addr);
|
|
|
|
void lldt(u_short sel);
|
2004-03-08 00:24:15 +00:00
|
|
|
void load_cr0(u_long cr0);
|
|
|
|
void load_cr3(u_long cr3);
|
|
|
|
void load_cr4(u_long cr4);
|
2011-04-14 16:14:35 +00:00
|
|
|
void load_dr0(uint64_t dr0);
|
|
|
|
void load_dr1(uint64_t dr1);
|
|
|
|
void load_dr2(uint64_t dr2);
|
|
|
|
void load_dr3(uint64_t dr3);
|
|
|
|
void load_dr6(uint64_t dr6);
|
|
|
|
void load_dr7(uint64_t dr7);
|
2010-09-03 14:25:17 +00:00
|
|
|
void load_fs(u_short sel);
|
|
|
|
void load_gs(u_short sel);
|
2002-09-22 04:45:21 +00:00
|
|
|
void ltr(u_short sel);
|
2002-03-20 19:04:56 +00:00
|
|
|
void outb(u_int port, u_char data);
|
|
|
|
void outl(u_int port, u_int data);
|
2010-01-01 20:55:11 +00:00
|
|
|
void outsb(u_int port, const void *addr, size_t count);
|
|
|
|
void outsl(u_int port, const void *addr, size_t count);
|
|
|
|
void outsw(u_int port, const void *addr, size_t count);
|
2002-03-20 19:04:56 +00:00
|
|
|
void outw(u_int port, u_short data);
|
2004-03-08 00:24:15 +00:00
|
|
|
u_long rcr0(void);
|
|
|
|
u_long rcr2(void);
|
|
|
|
u_long rcr3(void);
|
|
|
|
u_long rcr4(void);
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t rdmsr(u_int msr);
|
2015-02-09 21:00:56 +00:00
|
|
|
uint32_t rdmsr32(u_int msr);
|
2011-04-14 16:14:35 +00:00
|
|
|
uint64_t rdpmc(u_int pmc);
|
|
|
|
uint64_t rdr0(void);
|
|
|
|
uint64_t rdr1(void);
|
|
|
|
uint64_t rdr2(void);
|
|
|
|
uint64_t rdr3(void);
|
|
|
|
uint64_t rdr6(void);
|
|
|
|
uint64_t rdr7(void);
|
|
|
|
uint64_t rdtsc(void);
|
2012-02-27 17:28:47 +00:00
|
|
|
u_long read_rflags(void);
|
2004-03-08 00:24:15 +00:00
|
|
|
u_int rfs(void);
|
|
|
|
u_int rgs(void);
|
2002-03-20 19:04:56 +00:00
|
|
|
void wbinvd(void);
|
2003-05-01 01:05:25 +00:00
|
|
|
void write_rflags(u_int rf);
|
2011-04-14 16:14:35 +00:00
|
|
|
void wrmsr(u_int msr, uint64_t newval);
|
1994-11-14 15:04:06 +00:00
|
|
|
|
2005-03-02 21:33:29 +00:00
|
|
|
#endif /* __GNUCLIKE_ASM && __CC_SUPPORTS___INLINE */
|
1993-06-12 14:58:17 +00:00
|
|
|
|
2004-01-28 23:53:04 +00:00
|
|
|
void reset_dbregs(void);
|
2002-03-27 05:39:23 +00:00
|
|
|
|
2008-08-08 16:26:53 +00:00
|
|
|
#ifdef _KERNEL
|
|
|
|
int rdmsr_safe(u_int msr, uint64_t *val);
|
|
|
|
int wrmsr_safe(u_int msr, uint64_t newval);
|
|
|
|
#endif
|
|
|
|
|
1994-11-14 15:04:06 +00:00
|
|
|
#endif /* !_MACHINE_CPUFUNC_H_ */
|