2003-11-03 21:53:38 +00:00
|
|
|
/*-
|
|
|
|
* Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org>
|
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* Redistribution and use in source and binary forms, with or without
|
|
|
|
* modification, are permitted provided that the following conditions
|
|
|
|
* are met:
|
|
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer.
|
|
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
|
|
* documentation and/or other materials provided with the distribution.
|
|
|
|
*
|
|
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
|
|
|
|
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
|
|
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
|
|
|
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
|
|
|
|
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
|
|
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
|
|
|
|
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
|
|
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
|
|
|
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
|
|
|
|
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
|
|
|
|
* SUCH DAMAGE.
|
|
|
|
*
|
|
|
|
* $FreeBSD$
|
|
|
|
*/
|
|
|
|
|
2014-01-23 20:10:22 +00:00
|
|
|
#ifndef _X86_APICVAR_H_
|
|
|
|
#define _X86_APICVAR_H_
|
2003-11-03 21:53:38 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Local && I/O APIC variable definitions.
|
|
|
|
*/
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Layout of local APIC interrupt vectors:
|
|
|
|
*
|
|
|
|
* 0xff (255) +-------------+
|
2003-11-14 22:21:30 +00:00
|
|
|
* | | 15 (Spurious / IPIs / Local Interrupts)
|
2003-11-03 21:53:38 +00:00
|
|
|
* 0xf0 (240) +-------------+
|
2004-12-23 19:47:59 +00:00
|
|
|
* | | 14 (I/O Interrupts / Timer)
|
2003-11-03 21:53:38 +00:00
|
|
|
* 0xe0 (224) +-------------+
|
2003-11-14 19:10:13 +00:00
|
|
|
* | | 13 (I/O Interrupts)
|
2003-11-03 21:53:38 +00:00
|
|
|
* 0xd0 (208) +-------------+
|
2003-11-14 19:10:13 +00:00
|
|
|
* | | 12 (I/O Interrupts)
|
2003-11-03 21:53:38 +00:00
|
|
|
* 0xc0 (192) +-------------+
|
|
|
|
* | | 11 (I/O Interrupts)
|
|
|
|
* 0xb0 (176) +-------------+
|
|
|
|
* | | 10 (I/O Interrupts)
|
|
|
|
* 0xa0 (160) +-------------+
|
|
|
|
* | | 9 (I/O Interrupts)
|
|
|
|
* 0x90 (144) +-------------+
|
|
|
|
* | | 8 (I/O Interrupts / System Calls)
|
|
|
|
* 0x80 (128) +-------------+
|
|
|
|
* | | 7 (I/O Interrupts)
|
|
|
|
* 0x70 (112) +-------------+
|
|
|
|
* | | 6 (I/O Interrupts)
|
|
|
|
* 0x60 (96) +-------------+
|
|
|
|
* | | 5 (I/O Interrupts)
|
|
|
|
* 0x50 (80) +-------------+
|
|
|
|
* | | 4 (I/O Interrupts)
|
|
|
|
* 0x40 (64) +-------------+
|
|
|
|
* | | 3 (I/O Interrupts)
|
|
|
|
* 0x30 (48) +-------------+
|
2003-11-14 19:10:13 +00:00
|
|
|
* | | 2 (ATPIC Interrupts)
|
2003-11-03 21:53:38 +00:00
|
|
|
* 0x20 (32) +-------------+
|
|
|
|
* | | 1 (Exceptions, traps, faults, etc.)
|
|
|
|
* 0x10 (16) +-------------+
|
|
|
|
* | | 0 (Exceptions, traps, faults, etc.)
|
|
|
|
* 0x00 (0) +-------------+
|
|
|
|
*
|
|
|
|
* Note: 0x80 needs to be handled specially and not allocated to an
|
|
|
|
* I/O device!
|
|
|
|
*/
|
|
|
|
|
2007-05-08 22:01:04 +00:00
|
|
|
#define MAX_APIC_ID 0xfe
|
2003-11-03 21:53:38 +00:00
|
|
|
#define APIC_ID_ALL 0xff
|
2004-12-23 19:47:59 +00:00
|
|
|
|
|
|
|
/* I/O Interrupts are used for external devices such as ISA, PCI, etc. */
|
2003-11-14 19:10:13 +00:00
|
|
|
#define APIC_IO_INTS (IDT_IO_INTS + 16)
|
2004-12-23 19:47:59 +00:00
|
|
|
#define APIC_NUM_IOINTS 191
|
|
|
|
|
|
|
|
/* The timer interrupt is used for clock handling and drives hardclock, etc. */
|
|
|
|
#define APIC_TIMER_INT (APIC_IO_INTS + APIC_NUM_IOINTS)
|
2003-11-03 21:53:38 +00:00
|
|
|
|
2004-12-07 20:15:01 +00:00
|
|
|
/*
|
|
|
|
********************* !!! WARNING !!! ******************************
|
|
|
|
* Each local apic has an interrupt receive fifo that is two entries deep
|
|
|
|
* for each interrupt priority class (higher 4 bits of interrupt vector).
|
|
|
|
* Once the fifo is full the APIC can no longer receive interrupts for this
|
|
|
|
* class and sending IPIs from other CPUs will be blocked.
|
|
|
|
* To avoid deadlocks there should be no more than two IPI interrupts
|
|
|
|
* pending at the same time.
|
|
|
|
* Currently this is guaranteed by dividing the IPIs in two groups that have
|
|
|
|
* each at most one IPI interrupt pending. The first group is protected by the
|
|
|
|
* smp_ipi_mtx and waits for the completion of the IPI (Only one IPI user
|
|
|
|
* at a time) The second group uses a single interrupt and a bitmap to avoid
|
|
|
|
* redundant IPI interrupts.
|
|
|
|
*/
|
|
|
|
|
2004-12-23 19:47:59 +00:00
|
|
|
/* Interrupts for local APIC LVT entries other than the timer. */
|
2009-06-07 22:52:48 +00:00
|
|
|
#define APIC_LOCAL_INTS 240
|
|
|
|
#define APIC_ERROR_INT APIC_LOCAL_INTS
|
|
|
|
#define APIC_THERMAL_INT (APIC_LOCAL_INTS + 1)
|
2010-05-24 15:45:05 +00:00
|
|
|
#define APIC_CMC_INT (APIC_LOCAL_INTS + 2)
|
|
|
|
#define APIC_IPI_INTS (APIC_LOCAL_INTS + 3)
|
2009-06-07 22:52:48 +00:00
|
|
|
|
2004-12-07 20:15:01 +00:00
|
|
|
#define IPI_RENDEZVOUS (APIC_IPI_INTS) /* Inter-CPU rendezvous. */
|
2003-11-03 21:53:38 +00:00
|
|
|
#define IPI_INVLTLB (APIC_IPI_INTS + 1) /* TLB Shootdown IPIs */
|
|
|
|
#define IPI_INVLPG (APIC_IPI_INTS + 2)
|
|
|
|
#define IPI_INVLRNG (APIC_IPI_INTS + 3)
|
2006-05-01 21:36:47 +00:00
|
|
|
#define IPI_INVLCACHE (APIC_IPI_INTS + 4)
|
2004-12-07 20:15:01 +00:00
|
|
|
/* Vector to handle bitmap based IPIs */
|
2015-03-14 00:30:41 +00:00
|
|
|
#define IPI_BITMAP_VECTOR (APIC_IPI_INTS + 5)
|
2003-11-03 21:53:38 +00:00
|
|
|
|
2014-01-23 20:10:22 +00:00
|
|
|
/* IPIs handled by IPI_BITMAP_VECTOR */
|
2004-12-07 20:15:01 +00:00
|
|
|
#define IPI_AST 0 /* Generate software trap. */
|
2005-06-09 18:23:54 +00:00
|
|
|
#define IPI_PREEMPT 1
|
2014-01-23 20:10:22 +00:00
|
|
|
#define IPI_HARDCLOCK 2
|
Refactor timer management code with priority to one-shot operation mode.
The main goal of this is to generate timer interrupts only when there is
some work to do. When CPU is busy interrupts are generating at full rate
of hz + stathz to fullfill scheduler and timekeeping requirements. But
when CPU is idle, only minimum set of interrupts (down to 8 interrupts per
second per CPU now), needed to handle scheduled callouts is executed.
This allows significantly increase idle CPU sleep time, increasing effect
of static power-saving technologies. Also it should reduce host CPU load
on virtualized systems, when guest system is idle.
There is set of tunables, also available as writable sysctls, allowing to
control wanted event timer subsystem behavior:
kern.eventtimer.timer - allows to choose event timer hardware to use.
On x86 there is up to 4 different kinds of timers. Depending on whether
chosen timer is per-CPU, behavior of other options slightly differs.
kern.eventtimer.periodic - allows to choose periodic and one-shot
operation mode. In periodic mode, current timer hardware taken as the only
source of time for time events. This mode is quite alike to previous kernel
behavior. One-shot mode instead uses currently selected time counter
hardware to schedule all needed events one by one and program timer to
generate interrupt exactly in specified time. Default value depends of
chosen timer capabilities, but one-shot mode is preferred, until other is
forced by user or hardware.
kern.eventtimer.singlemul - in periodic mode specifies how much times
higher timer frequency should be, to not strictly alias hardclock() and
statclock() events. Default values are 2 and 4, but could be reduced to 1
if extra interrupts are unwanted.
kern.eventtimer.idletick - makes each CPU to receive every timer interrupt
independently of whether they busy or not. By default this options is
disabled. If chosen timer is per-CPU and runs in periodic mode, this option
has no effect - all interrupts are generating.
As soon as this patch modifies cpu_idle() on some platforms, I have also
refactored one on x86. Now it makes use of MONITOR/MWAIT instrunctions
(if supported) under high sleep/wakeup rate, as fast alternative to other
methods. It allows SMP scheduler to wake up sleeping CPUs much faster
without using IPI, significantly increasing performance on some highly
task-switching loads.
Tested by: many (on i386, amd64, sparc64 and powerc)
H/W donated by: Gheorghe Ardelean
Sponsored by: iXsystems, Inc.
2010-09-13 07:25:35 +00:00
|
|
|
#define IPI_BITMAP_LAST IPI_HARDCLOCK
|
2004-12-07 20:15:01 +00:00
|
|
|
#define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
|
|
|
|
|
2015-03-14 00:30:41 +00:00
|
|
|
#define IPI_STOP (APIC_IPI_INTS + 6) /* Stop CPU until restarted. */
|
|
|
|
#define IPI_SUSPEND (APIC_IPI_INTS + 7) /* Suspend CPU until restarted. */
|
|
|
|
#ifdef __i386__
|
|
|
|
#define IPI_LAZYPMAP (APIC_IPI_INTS + 8) /* Lazy pmap release. */
|
|
|
|
#define IPI_DYN_FIRST (APIC_IPI_INTS + 9)
|
|
|
|
#else
|
|
|
|
#define IPI_DYN_FIRST (APIC_IPI_INTS + 8)
|
|
|
|
#endif
|
2015-09-11 03:54:37 +00:00
|
|
|
#define IPI_DYN_LAST (253) /* IPIs allocated at runtime */
|
2015-03-01 02:31:27 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* IPI_STOP_HARD does not need to occupy a slot in the IPI vector space since
|
|
|
|
* it is delivered using an NMI anyways.
|
|
|
|
*/
|
2015-09-11 03:54:37 +00:00
|
|
|
#define IPI_NMI_FIRST 254
|
|
|
|
#define IPI_TRACE 254 /* Interrupt for tracing. */
|
2015-03-01 02:31:27 +00:00
|
|
|
#define IPI_STOP_HARD 255 /* Stop CPU with a NMI. */
|
2004-12-07 20:15:01 +00:00
|
|
|
|
2004-12-23 19:47:59 +00:00
|
|
|
/*
|
|
|
|
* The spurious interrupt can share the priority class with the IPIs since
|
2004-12-07 20:15:01 +00:00
|
|
|
* it is not a normal interrupt. (Does not use the APIC's interrupt fifo)
|
|
|
|
*/
|
2003-11-03 21:53:38 +00:00
|
|
|
#define APIC_SPURIOUS_INT 255
|
|
|
|
|
|
|
|
#ifndef LOCORE
|
|
|
|
|
|
|
|
#define APIC_IPI_DEST_SELF -1
|
|
|
|
#define APIC_IPI_DEST_ALL -2
|
|
|
|
#define APIC_IPI_DEST_OTHERS -3
|
|
|
|
|
2004-06-23 15:29:20 +00:00
|
|
|
#define APIC_BUS_UNKNOWN -1
|
|
|
|
#define APIC_BUS_ISA 0
|
|
|
|
#define APIC_BUS_EISA 1
|
|
|
|
#define APIC_BUS_PCI 2
|
|
|
|
#define APIC_BUS_MAX APIC_BUS_PCI
|
|
|
|
|
Use VT-d interrupt remapping block (IR) to perform FSB messages
translation. In particular, despite IO-APICs only take 8bit apic id,
IR translation structures accept 32bit APIC Id, which allows x2APIC
mode to function properly. Extend msi_cpu of struct msi_intrsrc and
io_cpu of ioapic_intsrc to full int from one byte.
KPI of IR is isolated into the x86/iommu/iommu_intrmap.h, to avoid
bringing all dmar headers into interrupt code. The non-PCI(e) devices
which generate message interrupts on FSB require special handling. The
HPET FSB interrupts are remapped, while DMAR interrupts are not.
For each msi and ioapic interrupt source, the iommu cookie is added,
which is in fact index of the IRE (interrupt remap entry) in the IR
table. Cookie is made at the source allocation time, and then used at
the map time to fill both IRE and device registers. The MSI
address/data registers and IO-APIC redirection registers are
programmed with the special values which are recognized by IR and used
to restore the IRE index, to find proper delivery mode and target.
Map all MSI interrupts in the block when msi_map() is called.
Since an interrupt source setup and dismantle code are done in the
non-sleepable context, flushing interrupt entries cache in the IR
hardware, which is done async and ideally waits for the interrupt,
requires busy-wait for queue to drain. The dmar_qi_wait_for_seq() is
modified to take a boolean argument requesting busy-wait for the
written sequence number instead of waiting for interrupt.
Some interrupts are configured before IR is initialized, e.g. ACPI
SCI. Add intr_reprogram() function to reprogram all already
configured interrupts, and call it immediately before an IR unit is
enabled. There is still a small window after the IO-APIC redirection
entry is reprogrammed with cookie but before the unit is enabled, but
to fix this properly, IR must be started much earlier.
Add workarounds for 5500 and X58 northbridges, some revisions of which
have severe flaws in handling IR. Use the same identification methods
as employed by Linux.
Review: https://reviews.freebsd.org/D1892
Reviewed by: neel
Discussed with: jhb
Tested by: glebius, pho (previous versions)
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
2015-03-19 13:57:47 +00:00
|
|
|
#define IRQ_EXTINT (NUM_IO_INTS + 1)
|
|
|
|
#define IRQ_NMI (NUM_IO_INTS + 2)
|
|
|
|
#define IRQ_SMI (NUM_IO_INTS + 3)
|
|
|
|
#define IRQ_DISABLED (NUM_IO_INTS + 4)
|
|
|
|
|
2003-11-03 21:53:38 +00:00
|
|
|
/*
|
|
|
|
* An APIC enumerator is a psuedo bus driver that enumerates APIC's including
|
|
|
|
* CPU's and I/O APIC's.
|
|
|
|
*/
|
|
|
|
struct apic_enumerator {
|
|
|
|
const char *apic_name;
|
|
|
|
int (*apic_probe)(void);
|
|
|
|
int (*apic_probe_cpus)(void);
|
|
|
|
int (*apic_setup_local)(void);
|
|
|
|
int (*apic_setup_io)(void);
|
|
|
|
SLIST_ENTRY(apic_enumerator) apic_next;
|
|
|
|
};
|
|
|
|
|
|
|
|
inthand_t
|
|
|
|
IDTVEC(apic_isr1), IDTVEC(apic_isr2), IDTVEC(apic_isr3),
|
2003-11-14 19:10:13 +00:00
|
|
|
IDTVEC(apic_isr4), IDTVEC(apic_isr5), IDTVEC(apic_isr6),
|
2010-05-24 15:45:05 +00:00
|
|
|
IDTVEC(apic_isr7), IDTVEC(cmcint), IDTVEC(errorint),
|
|
|
|
IDTVEC(spuriousint), IDTVEC(timerint);
|
2003-11-03 21:53:38 +00:00
|
|
|
|
2007-03-20 21:53:31 +00:00
|
|
|
extern vm_paddr_t lapic_paddr;
|
2009-01-29 09:22:56 +00:00
|
|
|
extern int apic_cpuids[];
|
|
|
|
|
2003-11-03 21:53:38 +00:00
|
|
|
void apic_register_enumerator(struct apic_enumerator *enumerator);
|
2007-03-05 20:35:17 +00:00
|
|
|
void *ioapic_create(vm_paddr_t addr, int32_t apic_id, int intbase);
|
2003-11-03 21:53:38 +00:00
|
|
|
int ioapic_disable_pin(void *cookie, u_int pin);
|
|
|
|
int ioapic_get_vector(void *cookie, u_int pin);
|
|
|
|
void ioapic_register(void *cookie);
|
|
|
|
int ioapic_remap_vector(void *cookie, u_int pin, int vector);
|
2004-06-23 15:29:20 +00:00
|
|
|
int ioapic_set_bus(void *cookie, u_int pin, int bus_type);
|
2003-11-03 21:53:38 +00:00
|
|
|
int ioapic_set_extint(void *cookie, u_int pin);
|
|
|
|
int ioapic_set_nmi(void *cookie, u_int pin);
|
2004-05-04 20:39:24 +00:00
|
|
|
int ioapic_set_polarity(void *cookie, u_int pin, enum intr_polarity pol);
|
|
|
|
int ioapic_set_triggermode(void *cookie, u_int pin,
|
|
|
|
enum intr_trigger trigger);
|
2003-11-03 21:53:38 +00:00
|
|
|
int ioapic_set_smi(void *cookie, u_int pin);
|
2014-06-16 08:43:03 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Struct containing pointers to APIC functions whose
|
|
|
|
* implementation is run time selectable.
|
|
|
|
*/
|
|
|
|
struct apic_ops {
|
|
|
|
void (*create)(u_int, int);
|
|
|
|
void (*init)(vm_paddr_t);
|
2015-02-09 21:00:56 +00:00
|
|
|
void (*xapic_mode)(void);
|
2014-06-16 08:43:03 +00:00
|
|
|
void (*setup)(int);
|
|
|
|
void (*dump)(const char *);
|
|
|
|
void (*disable)(void);
|
|
|
|
void (*eoi)(void);
|
|
|
|
int (*id)(void);
|
|
|
|
int (*intr_pending)(u_int);
|
|
|
|
void (*set_logical_id)(u_int, u_int, u_int);
|
|
|
|
u_int (*cpuid)(u_int);
|
|
|
|
|
|
|
|
/* Vectors */
|
|
|
|
u_int (*alloc_vector)(u_int, u_int);
|
|
|
|
u_int (*alloc_vectors)(u_int, u_int *, u_int, u_int);
|
|
|
|
void (*enable_vector)(u_int, u_int);
|
|
|
|
void (*disable_vector)(u_int, u_int);
|
|
|
|
void (*free_vector)(u_int, u_int, u_int);
|
|
|
|
|
|
|
|
|
|
|
|
/* PMC */
|
|
|
|
int (*enable_pmc)(void);
|
|
|
|
void (*disable_pmc)(void);
|
|
|
|
void (*reenable_pmc)(void);
|
|
|
|
|
|
|
|
/* CMC */
|
|
|
|
void (*enable_cmc)(void);
|
|
|
|
|
|
|
|
/* IPI */
|
|
|
|
void (*ipi_raw)(register_t, u_int);
|
|
|
|
void (*ipi_vectored)(u_int, int);
|
|
|
|
int (*ipi_wait)(int);
|
2015-03-14 00:30:41 +00:00
|
|
|
int (*ipi_alloc)(inthand_t *ipifunc);
|
|
|
|
void (*ipi_free)(int vector);
|
2014-06-16 08:43:03 +00:00
|
|
|
|
|
|
|
/* LVT */
|
|
|
|
int (*set_lvt_mask)(u_int, u_int, u_char);
|
|
|
|
int (*set_lvt_mode)(u_int, u_int, u_int32_t);
|
|
|
|
int (*set_lvt_polarity)(u_int, u_int, enum intr_polarity);
|
|
|
|
int (*set_lvt_triggermode)(u_int, u_int, enum intr_trigger);
|
|
|
|
};
|
|
|
|
|
|
|
|
extern struct apic_ops apic_ops;
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_create(u_int apic_id, int boot_cpu)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.create(apic_id, boot_cpu);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_init(vm_paddr_t addr)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.init(addr);
|
|
|
|
}
|
|
|
|
|
2015-02-09 21:00:56 +00:00
|
|
|
static inline void
|
|
|
|
lapic_xapic_mode(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.xapic_mode();
|
|
|
|
}
|
|
|
|
|
2014-06-16 08:43:03 +00:00
|
|
|
static inline void
|
|
|
|
lapic_setup(int boot)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.setup(boot);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_dump(const char *str)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.dump(str);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_disable(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.disable();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_eoi(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.eoi();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_id(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.id());
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_intr_pending(u_int vector)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.intr_pending(vector));
|
|
|
|
}
|
|
|
|
|
|
|
|
/* XXX: UNUSED */
|
|
|
|
static inline void
|
|
|
|
lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.set_logical_id(apic_id, cluster, cluster_id);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u_int
|
|
|
|
apic_cpuid(u_int apic_id)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.cpuid(apic_id));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u_int
|
|
|
|
apic_alloc_vector(u_int apic_id, u_int irq)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.alloc_vector(apic_id, irq));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline u_int
|
|
|
|
apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.alloc_vectors(apic_id, irqs, count, align));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
apic_enable_vector(u_int apic_id, u_int vector)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.enable_vector(apic_id, vector);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
apic_disable_vector(u_int apic_id, u_int vector)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.disable_vector(apic_id, vector);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
apic_free_vector(u_int apic_id, u_int vector, u_int irq)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.free_vector(apic_id, vector, irq);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_enable_pmc(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.enable_pmc());
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_disable_pmc(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.disable_pmc();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_reenable_pmc(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.reenable_pmc();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_enable_cmc(void)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.enable_cmc();
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_ipi_raw(register_t icrlo, u_int dest)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.ipi_raw(icrlo, dest);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_ipi_vectored(u_int vector, int dest)
|
|
|
|
{
|
|
|
|
|
|
|
|
apic_ops.ipi_vectored(vector, dest);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_ipi_wait(int delay)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.ipi_wait(delay));
|
|
|
|
}
|
|
|
|
|
2015-03-14 00:30:41 +00:00
|
|
|
static inline int
|
|
|
|
lapic_ipi_alloc(inthand_t *ipifunc)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.ipi_alloc(ipifunc));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void
|
|
|
|
lapic_ipi_free(int vector)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.ipi_free(vector));
|
|
|
|
}
|
|
|
|
|
2014-06-16 08:43:03 +00:00
|
|
|
static inline int
|
|
|
|
lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.set_lvt_mask(apic_id, lvt, masked));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_set_lvt_mode(u_int apic_id, u_int lvt, u_int32_t mode)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.set_lvt_mode(apic_id, lvt, mode));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.set_lvt_polarity(apic_id, lvt, pol));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline int
|
|
|
|
lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, enum intr_trigger trigger)
|
|
|
|
{
|
|
|
|
|
|
|
|
return (apic_ops.set_lvt_triggermode(apic_id, lvt, trigger));
|
|
|
|
}
|
|
|
|
|
2010-05-24 15:45:05 +00:00
|
|
|
void lapic_handle_cmc(void);
|
2010-03-29 19:13:34 +00:00
|
|
|
void lapic_handle_error(void);
|
2006-12-17 05:07:01 +00:00
|
|
|
void lapic_handle_intr(int vector, struct trapframe *frame);
|
|
|
|
void lapic_handle_timer(struct trapframe *frame);
|
Implement vector callback for PVHVM and unify event channel implementations
Re-structure Xen HVM support so that:
- Xen is detected and hypercalls can be performed very
early in system startup.
- Xen interrupt services are implemented using FreeBSD's native
interrupt delivery infrastructure.
- the Xen interrupt service implementation is shared between PV
and HVM guests.
- Xen interrupt handlers can optionally use a filter handler
in order to avoid the overhead of dispatch to an interrupt
thread.
- interrupt load can be distributed among all available CPUs.
- the overhead of accessing the emulated local and I/O apics
on HVM is removed for event channel port events.
- a similar optimization can eventually, and fairly easily,
be used to optimize MSI.
Early Xen detection, HVM refactoring, PVHVM interrupt infrastructure,
and misc Xen cleanups:
Sponsored by: Spectra Logic Corporation
Unification of PV & HVM interrupt infrastructure, bug fixes,
and misc Xen cleanups:
Submitted by: Roger Pau Monné
Sponsored by: Citrix Systems R&D
sys/x86/x86/local_apic.c:
sys/amd64/include/apicvar.h:
sys/i386/include/apicvar.h:
sys/amd64/amd64/apic_vector.S:
sys/i386/i386/apic_vector.s:
sys/amd64/amd64/machdep.c:
sys/i386/i386/machdep.c:
sys/i386/xen/exception.s:
sys/x86/include/segments.h:
Reserve IDT vector 0x93 for the Xen event channel upcall
interrupt handler. On Hypervisors that support the direct
vector callback feature, we can request that this vector be
called directly by an injected HVM interrupt event, instead
of a simulated PCI interrupt on the Xen platform PCI device.
This avoids all of the overhead of dealing with the emulated
I/O APIC and local APIC. It also means that the Hypervisor
can inject these events on any CPU, allowing upcalls for
different ports to be handled in parallel.
sys/amd64/amd64/mp_machdep.c:
sys/i386/i386/mp_machdep.c:
Map Xen per-vcpu area during AP startup.
sys/amd64/include/intr_machdep.h:
sys/i386/include/intr_machdep.h:
Increase the FreeBSD IRQ vector table to include space
for event channel interrupt sources.
sys/amd64/include/pcpu.h:
sys/i386/include/pcpu.h:
Remove Xen HVM per-cpu variable data. These fields are now
allocated via the dynamic per-cpu scheme. See xen_intr.c
for details.
sys/amd64/include/xen/hypercall.h:
sys/dev/xen/blkback/blkback.c:
sys/i386/include/xen/xenvar.h:
sys/i386/xen/clock.c:
sys/i386/xen/xen_machdep.c:
sys/xen/gnttab.c:
Prefer FreeBSD primatives to Linux ones in Xen support code.
sys/amd64/include/xen/xen-os.h:
sys/i386/include/xen/xen-os.h:
sys/xen/xen-os.h:
sys/dev/xen/balloon/balloon.c:
sys/dev/xen/blkback/blkback.c:
sys/dev/xen/blkfront/blkfront.c:
sys/dev/xen/console/xencons_ring.c:
sys/dev/xen/control/control.c:
sys/dev/xen/netback/netback.c:
sys/dev/xen/netfront/netfront.c:
sys/dev/xen/xenpci/xenpci.c:
sys/i386/i386/machdep.c:
sys/i386/include/pmap.h:
sys/i386/include/xen/xenfunc.h:
sys/i386/isa/npx.c:
sys/i386/xen/clock.c:
sys/i386/xen/mp_machdep.c:
sys/i386/xen/mptable.c:
sys/i386/xen/xen_clock_util.c:
sys/i386/xen/xen_machdep.c:
sys/i386/xen/xen_rtc.c:
sys/xen/evtchn/evtchn_dev.c:
sys/xen/features.c:
sys/xen/gnttab.c:
sys/xen/gnttab.h:
sys/xen/hvm.h:
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbus_if.m:
sys/xen/xenbus/xenbusb_front.c:
sys/xen/xenbus/xenbusvar.h:
sys/xen/xenstore/xenstore.c:
sys/xen/xenstore/xenstore_dev.c:
sys/xen/xenstore/xenstorevar.h:
Pull common Xen OS support functions/settings into xen/xen-os.h.
sys/amd64/include/xen/xen-os.h:
sys/i386/include/xen/xen-os.h:
sys/xen/xen-os.h:
Remove constants, macros, and functions unused in FreeBSD's Xen
support.
sys/xen/xen-os.h:
sys/i386/xen/xen_machdep.c:
sys/x86/xen/hvm.c:
Introduce new functions xen_domain(), xen_pv_domain(), and
xen_hvm_domain(). These are used in favor of #ifdefs so that
FreeBSD can dynamically detect and adapt to the presence of
a hypervisor. The goal is to have an HVM optimized GENERIC,
but more is necessary before this is possible.
sys/amd64/amd64/machdep.c:
sys/dev/xen/xenpci/xenpcivar.h:
sys/dev/xen/xenpci/xenpci.c:
sys/x86/xen/hvm.c:
sys/sys/kernel.h:
Refactor magic ioport, Hypercall table and Hypervisor shared
information page setup, and move it to a dedicated HVM support
module.
HVM mode initialization is now triggered during the
SI_SUB_HYPERVISOR phase of system startup. This currently
occurs just after the kernel VM is fully setup which is
just enough infrastructure to allow the hypercall table
and shared info page to be properly mapped.
sys/xen/hvm.h:
sys/x86/xen/hvm.c:
Add definitions and a method for configuring Hypervisor event
delievery via a direct vector callback.
sys/amd64/include/xen/xen-os.h:
sys/x86/xen/hvm.c:
sys/conf/files:
sys/conf/files.amd64:
sys/conf/files.i386:
Adjust kernel build to reflect the refactoring of early
Xen startup code and Xen interrupt services.
sys/dev/xen/blkback/blkback.c:
sys/dev/xen/blkfront/blkfront.c:
sys/dev/xen/blkfront/block.h:
sys/dev/xen/control/control.c:
sys/dev/xen/evtchn/evtchn_dev.c:
sys/dev/xen/netback/netback.c:
sys/dev/xen/netfront/netfront.c:
sys/xen/xenstore/xenstore.c:
sys/xen/evtchn/evtchn_dev.c:
sys/dev/xen/console/console.c:
sys/dev/xen/console/xencons_ring.c
Adjust drivers to use new xen_intr_*() API.
sys/dev/xen/blkback/blkback.c:
Since blkback defers all event handling to a taskqueue,
convert this task queue to a "fast" taskqueue, and schedule
it via an interrupt filter. This avoids an unnecessary
ithread context switch.
sys/xen/xenstore/xenstore.c:
The xenstore driver is MPSAFE. Indicate as much when
registering its interrupt handler.
sys/xen/xenbus/xenbus.c:
sys/xen/xenbus/xenbusvar.h:
Remove unused event channel APIs.
sys/xen/evtchn.h:
Remove all kernel Xen interrupt service API definitions
from this file. It is now only used for structure and
ioctl definitions related to the event channel userland
device driver.
Update the definitions in this file to match those from
NetBSD. Implementing this interface will be necessary for
Dom0 support.
sys/xen/evtchn/evtchnvar.h:
Add a header file for implemenation internal APIs related
to managing event channels event delivery. This is used
to allow, for example, the event channel userland device
driver to access low-level routines that typical kernel
consumers of event channel services should never access.
sys/xen/interface/event_channel.h:
sys/xen/xen_intr.h:
Standardize on the evtchn_port_t type for referring to
an event channel port id. In order to prevent low-level
event channel APIs from leaking to kernel consumers who
should not have access to this data, the type is defined
twice: Once in the Xen provided event_channel.h, and again
in xen/xen_intr.h. The double declaration is protected by
__XEN_EVTCHN_PORT_DEFINED__ to ensure it is never declared
twice within a given compilation unit.
sys/xen/xen_intr.h:
sys/xen/evtchn/evtchn.c:
sys/x86/xen/xen_intr.c:
sys/dev/xen/xenpci/evtchn.c:
sys/dev/xen/xenpci/xenpcivar.h:
New implementation of Xen interrupt services. This is
similar in many respects to the i386 PV implementation with
the exception that events for bound to event channel ports
(i.e. not IPI, virtual IRQ, or physical IRQ) are further
optimized to avoid mask/unmask operations that aren't
necessary for these edge triggered events.
Stubs exist for supporting physical IRQ binding, but will
need additional work before this implementation can be
fully shared between PV and HVM.
sys/amd64/amd64/mp_machdep.c:
sys/i386/i386/mp_machdep.c:
sys/i386/xen/mp_machdep.c
sys/x86/xen/hvm.c:
Add support for placing vcpu_info into an arbritary memory
page instead of using HYPERVISOR_shared_info->vcpu_info.
This allows the creation of domains with more than 32 vcpus.
sys/i386/i386/machdep.c:
sys/i386/xen/clock.c:
sys/i386/xen/xen_machdep.c:
sys/i386/xen/exception.s:
Add support for new event channle implementation.
2013-08-29 19:52:18 +00:00
|
|
|
void xen_intr_handle_upcall(struct trapframe *frame);
|
2015-04-29 10:12:34 +00:00
|
|
|
void hv_vector_handler(struct trapframe *frame);
|
2003-11-03 21:53:38 +00:00
|
|
|
|
2015-02-09 21:00:56 +00:00
|
|
|
extern int x2apic_mode;
|
2015-02-26 11:02:40 +00:00
|
|
|
extern int lapic_eoi_suppression;
|
2015-02-09 21:00:56 +00:00
|
|
|
|
|
|
|
#ifdef _SYS_SYSCTL_H_
|
|
|
|
SYSCTL_DECL(_hw_apic);
|
|
|
|
#endif
|
|
|
|
|
2003-11-03 21:53:38 +00:00
|
|
|
#endif /* !LOCORE */
|
2014-01-23 20:10:22 +00:00
|
|
|
#endif /* _X86_APICVAR_H_ */
|