d/freebsd-nq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
0f7c159f6b
freebsd-nq/sys/amd64/vmm/io/vlapic.c
Pedro F. Giffuni c49761dd57 sys/amd64: further adoption of SPDX licensing ID tags. 
Mainly focus on files that use BSD 2-Clause license, however the tool I
was using misidentified many licenses so this was mostly a manual - error
prone - task.

The Software Package Data Exchange (SPDX) group provides a specification
to make it easier for automated tools to detect and summarize well known
opensource licenses. We are gradually adopting the specification, noting
that the tags are considered only advisory and do not, in any way,
superceed or replace the license texts.
2017-11-27 15:03:07 +00:00

1657 lines
38 KiB
C
Raw Blame History

/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (c) 2011 NetApp, Inc.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/lock.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/systm.h>
#include <sys/smp.h>
#include <x86/specialreg.h>
#include <x86/apicreg.h>
#include <machine/clock.h>
#include <machine/smp.h>
#include <machine/vmm.h>
#include "vmm_lapic.h"
#include "vmm_ktr.h"
#include "vmm_stat.h"
#include "vlapic.h"
#include "vlapic_priv.h"
#include "vioapic.h"
#define PRIO(x) ((x) >> 4)
#define VLAPIC_VERSION (16)
#define x2apic(vlapic) (((vlapic)->msr_apicbase & APICBASE_X2APIC) ? 1 : 0)
/*
* The 'vlapic->timer_mtx' is used to provide mutual exclusion between the
* vlapic_callout_handler() and vcpu accesses to:
* - timer_freq_bt, timer_period_bt, timer_fire_bt
* - timer LVT register
*/
#define VLAPIC_TIMER_LOCK(vlapic) mtx_lock_spin(&((vlapic)->timer_mtx))
#define VLAPIC_TIMER_UNLOCK(vlapic) mtx_unlock_spin(&((vlapic)->timer_mtx))
#define VLAPIC_TIMER_LOCKED(vlapic) mtx_owned(&((vlapic)->timer_mtx))
/*
* APIC timer frequency:
* - arbitrary but chosen to be in the ballpark of contemporary hardware.
* - power-of-two to avoid loss of precision when converted to a bintime.
*/
#define VLAPIC_BUS_FREQ (128 * 1024 * 1024)
static __inline uint32_t
vlapic_get_id(struct vlapic *vlapic)
{
if (x2apic(vlapic))
return (vlapic->vcpuid);
else
return (vlapic->vcpuid << 24);
}
static uint32_t
x2apic_ldr(struct vlapic *vlapic)
{
int apicid;
uint32_t ldr;
apicid = vlapic_get_id(vlapic);
ldr = 1 << (apicid & 0xf);
ldr |= (apicid & 0xffff0) << 12;
return (ldr);
}
void
vlapic_dfr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
if (x2apic(vlapic)) {
VM_CTR1(vlapic->vm, "ignoring write to DFR in x2apic mode: %#x",
lapic->dfr);
lapic->dfr = 0;
return;
}
lapic->dfr &= APIC_DFR_MODEL_MASK;
lapic->dfr |= APIC_DFR_RESERVED;
if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_FLAT)
VLAPIC_CTR0(vlapic, "vlapic DFR in Flat Model");
else if ((lapic->dfr & APIC_DFR_MODEL_MASK) == APIC_DFR_MODEL_CLUSTER)
VLAPIC_CTR0(vlapic, "vlapic DFR in Cluster Model");
else
VLAPIC_CTR1(vlapic, "DFR in Unknown Model %#x", lapic->dfr);
}
void
vlapic_ldr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
/* LDR is read-only in x2apic mode */
if (x2apic(vlapic)) {
VLAPIC_CTR1(vlapic, "ignoring write to LDR in x2apic mode: %#x",
lapic->ldr);
lapic->ldr = x2apic_ldr(vlapic);
} else {
lapic->ldr &= ~APIC_LDR_RESERVED;
VLAPIC_CTR1(vlapic, "vlapic LDR set to %#x", lapic->ldr);
}
}
void
vlapic_id_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
/*
* We don't allow the ID register to be modified so reset it back to
* its default value.
*/
lapic = vlapic->apic_page;
lapic->id = vlapic_get_id(vlapic);
}
static int
vlapic_timer_divisor(uint32_t dcr)
{
switch (dcr & 0xB) {
case APIC_TDCR_1:
return (1);
case APIC_TDCR_2:
return (2);
case APIC_TDCR_4:
return (4);
case APIC_TDCR_8:
return (8);
case APIC_TDCR_16:
return (16);
case APIC_TDCR_32:
return (32);
case APIC_TDCR_64:
return (64);
case APIC_TDCR_128:
return (128);
default:
panic("vlapic_timer_divisor: invalid dcr 0x%08x", dcr);
}
}
#if 0
static inline void
vlapic_dump_lvt(uint32_t offset, uint32_t *lvt)
{
printf("Offset %x: lvt %08x (V:%02x DS:%x M:%x)\n", offset,
*lvt, *lvt & APIC_LVTT_VECTOR, *lvt & APIC_LVTT_DS,
*lvt & APIC_LVTT_M);
}
#endif
static uint32_t
vlapic_get_ccr(struct vlapic *vlapic)
{
struct bintime bt_now, bt_rem;
struct LAPIC *lapic;
uint32_t ccr;
ccr = 0;
lapic = vlapic->apic_page;
VLAPIC_TIMER_LOCK(vlapic);
if (callout_active(&vlapic->callout)) {
/*
* If the timer is scheduled to expire in the future then
* compute the value of 'ccr' based on the remaining time.
*/
binuptime(&bt_now);
if (bintime_cmp(&vlapic->timer_fire_bt, &bt_now, >)) {
bt_rem = vlapic->timer_fire_bt;
bintime_sub(&bt_rem, &bt_now);
ccr += bt_rem.sec * BT2FREQ(&vlapic->timer_freq_bt);
ccr += bt_rem.frac / vlapic->timer_freq_bt.frac;
}
}
KASSERT(ccr <= lapic->icr_timer, ("vlapic_get_ccr: invalid ccr %#x, "
"icr_timer is %#x", ccr, lapic->icr_timer));
VLAPIC_CTR2(vlapic, "vlapic ccr_timer = %#x, icr_timer = %#x",
ccr, lapic->icr_timer);
VLAPIC_TIMER_UNLOCK(vlapic);
return (ccr);
}
void
vlapic_dcr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
int divisor;
lapic = vlapic->apic_page;
VLAPIC_TIMER_LOCK(vlapic);
divisor = vlapic_timer_divisor(lapic->dcr_timer);
VLAPIC_CTR2(vlapic, "vlapic dcr_timer=%#x, divisor=%d",
lapic->dcr_timer, divisor);
/*
* Update the timer frequency and the timer period.
*
* XXX changes to the frequency divider will not take effect until
* the timer is reloaded.
*/
FREQ2BT(VLAPIC_BUS_FREQ / divisor, &vlapic->timer_freq_bt);
vlapic->timer_period_bt = vlapic->timer_freq_bt;
bintime_mul(&vlapic->timer_period_bt, lapic->icr_timer);
VLAPIC_TIMER_UNLOCK(vlapic);
}
void
vlapic_esr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
lapic->esr = vlapic->esr_pending;
vlapic->esr_pending = 0;
}
int
vlapic_set_intr_ready(struct vlapic *vlapic, int vector, bool level)
{
struct LAPIC *lapic;
uint32_t *irrptr, *tmrptr, mask;
int idx;
KASSERT(vector >= 0 && vector < 256, ("invalid vector %d", vector));
lapic = vlapic->apic_page;
if (!(lapic->svr & APIC_SVR_ENABLE)) {
VLAPIC_CTR1(vlapic, "vlapic is software disabled, ignoring "
"interrupt %d", vector);
return (0);
}
if (vector < 16) {
vlapic_set_error(vlapic, APIC_ESR_RECEIVE_ILLEGAL_VECTOR);
VLAPIC_CTR1(vlapic, "vlapic ignoring interrupt to vector %d",
vector);
return (1);
}
if (vlapic->ops.set_intr_ready)
return ((*vlapic->ops.set_intr_ready)(vlapic, vector, level));
idx = (vector / 32) * 4;
mask = 1 << (vector % 32);
irrptr = &lapic->irr0;
atomic_set_int(&irrptr[idx], mask);
/*
* Verify that the trigger-mode of the interrupt matches with
* the vlapic TMR registers.
*/
tmrptr = &lapic->tmr0;
if ((tmrptr[idx] & mask) != (level ? mask : 0)) {
VLAPIC_CTR3(vlapic, "vlapic TMR[%d] is 0x%08x but "
"interrupt is %s-triggered", idx / 4, tmrptr[idx],
level ? "level" : "edge");
}
VLAPIC_CTR_IRR(vlapic, "vlapic_set_intr_ready");
return (1);
}
static __inline uint32_t *
vlapic_get_lvtptr(struct vlapic *vlapic, uint32_t offset)
{
struct LAPIC *lapic = vlapic->apic_page;
int i;
switch (offset) {
case APIC_OFFSET_CMCI_LVT:
return (&lapic->lvt_cmci);
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
i = (offset - APIC_OFFSET_TIMER_LVT) >> 2;
return ((&lapic->lvt_timer) + i);;
default:
panic("vlapic_get_lvt: invalid LVT\n");
}
}
static __inline int
lvt_off_to_idx(uint32_t offset)
{
int index;
switch (offset) {
case APIC_OFFSET_CMCI_LVT:
index = APIC_LVT_CMCI;
break;
case APIC_OFFSET_TIMER_LVT:
index = APIC_LVT_TIMER;
break;
case APIC_OFFSET_THERM_LVT:
index = APIC_LVT_THERMAL;
break;
case APIC_OFFSET_PERF_LVT:
index = APIC_LVT_PMC;
break;
case APIC_OFFSET_LINT0_LVT:
index = APIC_LVT_LINT0;
break;
case APIC_OFFSET_LINT1_LVT:
index = APIC_LVT_LINT1;
break;
case APIC_OFFSET_ERROR_LVT:
index = APIC_LVT_ERROR;
break;
default:
index = -1;
break;
}
KASSERT(index >= 0 && index <= VLAPIC_MAXLVT_INDEX, ("lvt_off_to_idx: "
"invalid lvt index %d for offset %#x", index, offset));
return (index);
}
static __inline uint32_t
vlapic_get_lvt(struct vlapic *vlapic, uint32_t offset)
{
int idx;
uint32_t val;
idx = lvt_off_to_idx(offset);
val = atomic_load_acq_32(&vlapic->lvt_last[idx]);
return (val);
}
void
vlapic_lvt_write_handler(struct vlapic *vlapic, uint32_t offset)
{
uint32_t *lvtptr, mask, val;
struct LAPIC *lapic;
int idx;
lapic = vlapic->apic_page;
lvtptr = vlapic_get_lvtptr(vlapic, offset);
val = *lvtptr;
idx = lvt_off_to_idx(offset);
if (!(lapic->svr & APIC_SVR_ENABLE))
val |= APIC_LVT_M;
mask = APIC_LVT_M | APIC_LVT_DS | APIC_LVT_VECTOR;
switch (offset) {
case APIC_OFFSET_TIMER_LVT:
mask |= APIC_LVTT_TM;
break;
case APIC_OFFSET_ERROR_LVT:
break;
case APIC_OFFSET_LINT0_LVT:
case APIC_OFFSET_LINT1_LVT:
mask |= APIC_LVT_TM | APIC_LVT_RIRR | APIC_LVT_IIPP;
/* FALLTHROUGH */
default:
mask |= APIC_LVT_DM;
break;
}
val &= mask;
*lvtptr = val;
atomic_store_rel_32(&vlapic->lvt_last[idx], val);
}
static void
vlapic_mask_lvts(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
lapic->lvt_cmci |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_CMCI_LVT);
lapic->lvt_timer |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_TIMER_LVT);
lapic->lvt_thermal |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_THERM_LVT);
lapic->lvt_pcint |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_PERF_LVT);
lapic->lvt_lint0 |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT0_LVT);
lapic->lvt_lint1 |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_LINT1_LVT);
lapic->lvt_error |= APIC_LVT_M;
vlapic_lvt_write_handler(vlapic, APIC_OFFSET_ERROR_LVT);
}
static int
vlapic_fire_lvt(struct vlapic *vlapic, uint32_t lvt)
{
uint32_t vec, mode;
if (lvt & APIC_LVT_M)
return (0);
vec = lvt & APIC_LVT_VECTOR;
mode = lvt & APIC_LVT_DM;
switch (mode) {
case APIC_LVT_DM_FIXED:
if (vec < 16) {
vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
return (0);
}
if (vlapic_set_intr_ready(vlapic, vec, false))
vcpu_notify_event(vlapic->vm, vlapic->vcpuid, true);
break;
case APIC_LVT_DM_NMI:
vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
break;
case APIC_LVT_DM_EXTINT:
vm_inject_extint(vlapic->vm, vlapic->vcpuid);
break;
default:
// Other modes ignored
return (0);
}
return (1);
}
#if 1
static void
dump_isrvec_stk(struct vlapic *vlapic)
{
int i;
uint32_t *isrptr;
isrptr = &vlapic->apic_page->isr0;
for (i = 0; i < 8; i++)
printf("ISR%d 0x%08x\n", i, isrptr[i * 4]);
for (i = 0; i <= vlapic->isrvec_stk_top; i++)
printf("isrvec_stk[%d] = %d\n", i, vlapic->isrvec_stk[i]);
}
#endif
/*
* Algorithm adopted from section "Interrupt, Task and Processor Priority"
* in Intel Architecture Manual Vol 3a.
*/
static void
vlapic_update_ppr(struct vlapic *vlapic)
{
int isrvec, tpr, ppr;
/*
* Note that the value on the stack at index 0 is always 0.
*
* This is a placeholder for the value of ISRV when none of the
* bits is set in the ISRx registers.
*/
isrvec = vlapic->isrvec_stk[vlapic->isrvec_stk_top];
tpr = vlapic->apic_page->tpr;
#if 1
{
int i, lastprio, curprio, vector, idx;
uint32_t *isrptr;
if (vlapic->isrvec_stk_top == 0 && isrvec != 0)
panic("isrvec_stk is corrupted: %d", isrvec);
/*
* Make sure that the priority of the nested interrupts is
* always increasing.
*/
lastprio = -1;
for (i = 1; i <= vlapic->isrvec_stk_top; i++) {
curprio = PRIO(vlapic->isrvec_stk[i]);
if (curprio <= lastprio) {
dump_isrvec_stk(vlapic);
panic("isrvec_stk does not satisfy invariant");
}
lastprio = curprio;
}
/*
* Make sure that each bit set in the ISRx registers has a
* corresponding entry on the isrvec stack.
*/
i = 1;
isrptr = &vlapic->apic_page->isr0;
for (vector = 0; vector < 256; vector++) {
idx = (vector / 32) * 4;
if (isrptr[idx] & (1 << (vector % 32))) {
if (i > vlapic->isrvec_stk_top ||
vlapic->isrvec_stk[i] != vector) {
dump_isrvec_stk(vlapic);
panic("ISR and isrvec_stk out of sync");
}
i++;
}
}
}
#endif
if (PRIO(tpr) >= PRIO(isrvec))
ppr = tpr;
else
ppr = isrvec & 0xf0;
vlapic->apic_page->ppr = ppr;
VLAPIC_CTR1(vlapic, "vlapic_update_ppr 0x%02x", ppr);
}
static VMM_STAT(VLAPIC_GRATUITOUS_EOI, "EOI without any in-service interrupt");
static void
vlapic_process_eoi(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *isrptr, *tmrptr;
int i, idx, bitpos, vector;
isrptr = &lapic->isr0;
tmrptr = &lapic->tmr0;
for (i = 7; i >= 0; i--) {
idx = i * 4;
bitpos = fls(isrptr[idx]);
if (bitpos-- != 0) {
if (vlapic->isrvec_stk_top <= 0) {
panic("invalid vlapic isrvec_stk_top %d",
vlapic->isrvec_stk_top);
}
isrptr[idx] &= ~(1 << bitpos);
vector = i * 32 + bitpos;
VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "EOI vector %d",
vector);
VLAPIC_CTR_ISR(vlapic, "vlapic_process_eoi");
vlapic->isrvec_stk_top--;
vlapic_update_ppr(vlapic);
if ((tmrptr[idx] & (1 << bitpos)) != 0) {
vioapic_process_eoi(vlapic->vm, vlapic->vcpuid,
vector);
}
return;
}
}
VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "Gratuitous EOI");
vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_GRATUITOUS_EOI, 1);
}
static __inline int
vlapic_get_lvt_field(uint32_t lvt, uint32_t mask)
{
return (lvt & mask);
}
static __inline int
vlapic_periodic_timer(struct vlapic *vlapic)
{
uint32_t lvt;
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
return (vlapic_get_lvt_field(lvt, APIC_LVTT_TM_PERIODIC));
}
static VMM_STAT(VLAPIC_INTR_ERROR, "error interrupts generated by vlapic");
void
vlapic_set_error(struct vlapic *vlapic, uint32_t mask)
{
uint32_t lvt;
vlapic->esr_pending |= mask;
if (vlapic->esr_firing)
return;
vlapic->esr_firing = 1;
// The error LVT always uses the fixed delivery mode.
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_ERROR, 1);
}
vlapic->esr_firing = 0;
}
static VMM_STAT(VLAPIC_INTR_TIMER, "timer interrupts generated by vlapic");
static void
vlapic_fire_timer(struct vlapic *vlapic)
{
uint32_t lvt;
KASSERT(VLAPIC_TIMER_LOCKED(vlapic), ("vlapic_fire_timer not locked"));
// The timer LVT always uses the fixed delivery mode.
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
if (vlapic_fire_lvt(vlapic, lvt | APIC_LVT_DM_FIXED)) {
VLAPIC_CTR0(vlapic, "vlapic timer fired");
vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_TIMER, 1);
}
}
static VMM_STAT(VLAPIC_INTR_CMC,
"corrected machine check interrupts generated by vlapic");
void
vlapic_fire_cmci(struct vlapic *vlapic)
{
uint32_t lvt;
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
if (vlapic_fire_lvt(vlapic, lvt)) {
vmm_stat_incr(vlapic->vm, vlapic->vcpuid, VLAPIC_INTR_CMC, 1);
}
}
static VMM_STAT_ARRAY(LVTS_TRIGGERRED, VLAPIC_MAXLVT_INDEX + 1,
"lvts triggered");
int
vlapic_trigger_lvt(struct vlapic *vlapic, int vector)
{
uint32_t lvt;
if (vlapic_enabled(vlapic) == false) {
/*
* When the local APIC is global/hardware disabled,
* LINT[1:0] pins are configured as INTR and NMI pins,
* respectively.
*/
switch (vector) {
case APIC_LVT_LINT0:
vm_inject_extint(vlapic->vm, vlapic->vcpuid);
break;
case APIC_LVT_LINT1:
vm_inject_nmi(vlapic->vm, vlapic->vcpuid);
break;
default:
break;
}
return (0);
}
switch (vector) {
case APIC_LVT_LINT0:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT0_LVT);
break;
case APIC_LVT_LINT1:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_LINT1_LVT);
break;
case APIC_LVT_TIMER:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_TIMER_LVT);
lvt |= APIC_LVT_DM_FIXED;
break;
case APIC_LVT_ERROR:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_ERROR_LVT);
lvt |= APIC_LVT_DM_FIXED;
break;
case APIC_LVT_PMC:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_PERF_LVT);
break;
case APIC_LVT_THERMAL:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_THERM_LVT);
break;
case APIC_LVT_CMCI:
lvt = vlapic_get_lvt(vlapic, APIC_OFFSET_CMCI_LVT);
break;
default:
return (EINVAL);
}
if (vlapic_fire_lvt(vlapic, lvt)) {
vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
LVTS_TRIGGERRED, vector, 1);
}
return (0);
}
static void
vlapic_callout_handler(void *arg)
{
struct vlapic *vlapic;
struct bintime bt, btnow;
sbintime_t rem_sbt;
vlapic = arg;
VLAPIC_TIMER_LOCK(vlapic);
if (callout_pending(&vlapic->callout)) /* callout was reset */
goto done;
if (!callout_active(&vlapic->callout)) /* callout was stopped */
goto done;
callout_deactivate(&vlapic->callout);
vlapic_fire_timer(vlapic);
if (vlapic_periodic_timer(vlapic)) {
binuptime(&btnow);
KASSERT(bintime_cmp(&btnow, &vlapic->timer_fire_bt, >=),
("vlapic callout at %#lx.%#lx, expected at %#lx.#%lx",
btnow.sec, btnow.frac, vlapic->timer_fire_bt.sec,
vlapic->timer_fire_bt.frac));
/*
* Compute the delta between when the timer was supposed to
* fire and the present time.
*/
bt = btnow;
bintime_sub(&bt, &vlapic->timer_fire_bt);
rem_sbt = bttosbt(vlapic->timer_period_bt);
if (bintime_cmp(&bt, &vlapic->timer_period_bt, <)) {
/*
* Adjust the time until the next countdown downward
* to account for the lost time.
*/
rem_sbt -= bttosbt(bt);
} else {
/*
* If the delta is greater than the timer period then
* just reset our time base instead of trying to catch
* up.
*/
vlapic->timer_fire_bt = btnow;
VLAPIC_CTR2(vlapic, "vlapic timer lagging by %lu "
"usecs, period is %lu usecs - resetting time base",
bttosbt(bt) / SBT_1US,
bttosbt(vlapic->timer_period_bt) / SBT_1US);
}
bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
callout_reset_sbt(&vlapic->callout, rem_sbt, 0,
vlapic_callout_handler, vlapic, 0);
}
done:
VLAPIC_TIMER_UNLOCK(vlapic);
}
void
vlapic_icrtmr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
sbintime_t sbt;
uint32_t icr_timer;
VLAPIC_TIMER_LOCK(vlapic);
lapic = vlapic->apic_page;
icr_timer = lapic->icr_timer;
vlapic->timer_period_bt = vlapic->timer_freq_bt;
bintime_mul(&vlapic->timer_period_bt, icr_timer);
if (icr_timer != 0) {
binuptime(&vlapic->timer_fire_bt);
bintime_add(&vlapic->timer_fire_bt, &vlapic->timer_period_bt);
sbt = bttosbt(vlapic->timer_period_bt);
callout_reset_sbt(&vlapic->callout, sbt, 0,
vlapic_callout_handler, vlapic, 0);
} else
callout_stop(&vlapic->callout);
VLAPIC_TIMER_UNLOCK(vlapic);
}
/*
* This function populates 'dmask' with the set of vcpus that match the
* addressing specified by the (dest, phys, lowprio) tuple.
*
* 'x2apic_dest' specifies whether 'dest' is interpreted as x2APIC (32-bit)
* or xAPIC (8-bit) destination field.
*/
static void
vlapic_calcdest(struct vm *vm, cpuset_t *dmask, uint32_t dest, bool phys,
bool lowprio, bool x2apic_dest)
{
struct vlapic *vlapic;
uint32_t dfr, ldr, ldest, cluster;
uint32_t mda_flat_ldest, mda_cluster_ldest, mda_ldest, mda_cluster_id;
cpuset_t amask;
int vcpuid;
if ((x2apic_dest && dest == 0xffffffff) ||
(!x2apic_dest && dest == 0xff)) {
/*
* Broadcast in both logical and physical modes.
*/
*dmask = vm_active_cpus(vm);
return;
}
if (phys) {
/*
* Physical mode: destination is APIC ID.
*/
CPU_ZERO(dmask);
vcpuid = vm_apicid2vcpuid(vm, dest);
if (vcpuid < VM_MAXCPU)
CPU_SET(vcpuid, dmask);
} else {
/*
* In the "Flat Model" the MDA is interpreted as an 8-bit wide
* bitmask. This model is only available in the xAPIC mode.
*/
mda_flat_ldest = dest & 0xff;
/*
* In the "Cluster Model" the MDA is used to identify a
* specific cluster and a set of APICs in that cluster.
*/
if (x2apic_dest) {
mda_cluster_id = dest >> 16;
mda_cluster_ldest = dest & 0xffff;
} else {
mda_cluster_id = (dest >> 4) & 0xf;
mda_cluster_ldest = dest & 0xf;
}
/*
* Logical mode: match each APIC that has a bit set
* in its LDR that matches a bit in the ldest.
*/
CPU_ZERO(dmask);
amask = vm_active_cpus(vm);
while ((vcpuid = CPU_FFS(&amask)) != 0) {
vcpuid--;
CPU_CLR(vcpuid, &amask);
vlapic = vm_lapic(vm, vcpuid);
dfr = vlapic->apic_page->dfr;
ldr = vlapic->apic_page->ldr;
if ((dfr & APIC_DFR_MODEL_MASK) ==
APIC_DFR_MODEL_FLAT) {
ldest = ldr >> 24;
mda_ldest = mda_flat_ldest;
} else if ((dfr & APIC_DFR_MODEL_MASK) ==
APIC_DFR_MODEL_CLUSTER) {
if (x2apic(vlapic)) {
cluster = ldr >> 16;
ldest = ldr & 0xffff;
} else {
cluster = ldr >> 28;
ldest = (ldr >> 24) & 0xf;
}
if (cluster != mda_cluster_id)
continue;
mda_ldest = mda_cluster_ldest;
} else {
/*
* Guest has configured a bad logical
* model for this vcpu - skip it.
*/
VLAPIC_CTR1(vlapic, "vlapic has bad logical "
"model %x - cannot deliver interrupt", dfr);
continue;
}
if ((mda_ldest & ldest) != 0) {
CPU_SET(vcpuid, dmask);
if (lowprio)
break;
}
}
}
}
static VMM_STAT_ARRAY(IPIS_SENT, VM_MAXCPU, "ipis sent to vcpu");
static void
vlapic_set_tpr(struct vlapic *vlapic, uint8_t val)
{
struct LAPIC *lapic = vlapic->apic_page;
if (lapic->tpr != val) {
VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vlapic TPR changed "
"from %#x to %#x", lapic->tpr, val);
lapic->tpr = val;
vlapic_update_ppr(vlapic);
}
}
static uint8_t
vlapic_get_tpr(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
return (lapic->tpr);
}
void
vlapic_set_cr8(struct vlapic *vlapic, uint64_t val)
{
uint8_t tpr;
if (val & ~0xf) {
vm_inject_gp(vlapic->vm, vlapic->vcpuid);
return;
}
tpr = val << 4;
vlapic_set_tpr(vlapic, tpr);
}
uint64_t
vlapic_get_cr8(struct vlapic *vlapic)
{
uint8_t tpr;
tpr = vlapic_get_tpr(vlapic);
return (tpr >> 4);
}
int
vlapic_icrlo_write_handler(struct vlapic *vlapic, bool *retu)
{
int i;
bool phys;
cpuset_t dmask;
uint64_t icrval;
uint32_t dest, vec, mode;
struct vlapic *vlapic2;
struct vm_exit *vmexit;
struct LAPIC *lapic;
lapic = vlapic->apic_page;
lapic->icr_lo &= ~APIC_DELSTAT_PEND;
icrval = ((uint64_t)lapic->icr_hi << 32) | lapic->icr_lo;
if (x2apic(vlapic))
dest = icrval >> 32;
else
dest = icrval >> (32 + 24);
vec = icrval & APIC_VECTOR_MASK;
mode = icrval & APIC_DELMODE_MASK;
if (mode == APIC_DELMODE_FIXED && vec < 16) {
vlapic_set_error(vlapic, APIC_ESR_SEND_ILLEGAL_VECTOR);
VLAPIC_CTR1(vlapic, "Ignoring invalid IPI %d", vec);
return (0);
}
VLAPIC_CTR2(vlapic, "icrlo 0x%016lx triggered ipi %d", icrval, vec);
if (mode == APIC_DELMODE_FIXED || mode == APIC_DELMODE_NMI) {
switch (icrval & APIC_DEST_MASK) {
case APIC_DEST_DESTFLD:
phys = ((icrval & APIC_DESTMODE_LOG) == 0);
vlapic_calcdest(vlapic->vm, &dmask, dest, phys, false,
x2apic(vlapic));
break;
case APIC_DEST_SELF:
CPU_SETOF(vlapic->vcpuid, &dmask);
break;
case APIC_DEST_ALLISELF:
dmask = vm_active_cpus(vlapic->vm);
break;
case APIC_DEST_ALLESELF:
dmask = vm_active_cpus(vlapic->vm);
CPU_CLR(vlapic->vcpuid, &dmask);
break;
default:
CPU_ZERO(&dmask); /* satisfy gcc */
break;
}
while ((i = CPU_FFS(&dmask)) != 0) {
i--;
CPU_CLR(i, &dmask);
if (mode == APIC_DELMODE_FIXED) {
lapic_intr_edge(vlapic->vm, i, vec);
vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid,
IPIS_SENT, i, 1);
VLAPIC_CTR2(vlapic, "vlapic sending ipi %d "
"to vcpuid %d", vec, i);
} else {
vm_inject_nmi(vlapic->vm, i);
VLAPIC_CTR1(vlapic, "vlapic sending ipi nmi "
"to vcpuid %d", i);
}
}
return (0); /* handled completely in the kernel */
}
if (mode == APIC_DELMODE_INIT) {
if ((icrval & APIC_LEVEL_MASK) == APIC_LEVEL_DEASSERT)
return (0);
if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
vlapic2 = vm_lapic(vlapic->vm, dest);
/* move from INIT to waiting-for-SIPI state */
if (vlapic2->boot_state == BS_INIT) {
vlapic2->boot_state = BS_SIPI;
}
return (0);
}
}
if (mode == APIC_DELMODE_STARTUP) {
if (vlapic->vcpuid == 0 && dest != 0 && dest < VM_MAXCPU) {
vlapic2 = vm_lapic(vlapic->vm, dest);
/*
* Ignore SIPIs in any state other than wait-for-SIPI
*/
if (vlapic2->boot_state != BS_SIPI)
return (0);
vlapic2->boot_state = BS_RUNNING;
*retu = true;
vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid);
vmexit->exitcode = VM_EXITCODE_SPINUP_AP;
vmexit->u.spinup_ap.vcpu = dest;
vmexit->u.spinup_ap.rip = vec << PAGE_SHIFT;
return (0);
}
}
/*
* This will cause a return to userland.
*/
return (1);
}
void
vlapic_self_ipi_handler(struct vlapic *vlapic, uint64_t val)
{
int vec;
KASSERT(x2apic(vlapic), ("SELF_IPI does not exist in xAPIC mode"));
vec = val & 0xff;
lapic_intr_edge(vlapic->vm, vlapic->vcpuid, vec);
vmm_stat_array_incr(vlapic->vm, vlapic->vcpuid, IPIS_SENT,
vlapic->vcpuid, 1);
VLAPIC_CTR1(vlapic, "vlapic self-ipi %d", vec);
}
int
vlapic_pending_intr(struct vlapic *vlapic, int *vecptr)
{
struct LAPIC *lapic = vlapic->apic_page;
int idx, i, bitpos, vector;
uint32_t *irrptr, val;
if (vlapic->ops.pending_intr)
return ((*vlapic->ops.pending_intr)(vlapic, vecptr));
irrptr = &lapic->irr0;
for (i = 7; i >= 0; i--) {
idx = i * 4;
val = atomic_load_acq_int(&irrptr[idx]);
bitpos = fls(val);
if (bitpos != 0) {
vector = i * 32 + (bitpos - 1);
if (PRIO(vector) > PRIO(lapic->ppr)) {
VLAPIC_CTR1(vlapic, "pending intr %d", vector);
if (vecptr != NULL)
*vecptr = vector;
return (1);
} else
break;
}
}
return (0);
}
void
vlapic_intr_accepted(struct vlapic *vlapic, int vector)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *irrptr, *isrptr;
int idx, stk_top;
if (vlapic->ops.intr_accepted)
return ((*vlapic->ops.intr_accepted)(vlapic, vector));
/*
* clear the ready bit for vector being accepted in irr
* and set the vector as in service in isr.
*/
idx = (vector / 32) * 4;
irrptr = &lapic->irr0;
atomic_clear_int(&irrptr[idx], 1 << (vector % 32));
VLAPIC_CTR_IRR(vlapic, "vlapic_intr_accepted");
isrptr = &lapic->isr0;
isrptr[idx] |= 1 << (vector % 32);
VLAPIC_CTR_ISR(vlapic, "vlapic_intr_accepted");
/*
* Update the PPR
*/
vlapic->isrvec_stk_top++;
stk_top = vlapic->isrvec_stk_top;
if (stk_top >= ISRVEC_STK_SIZE)
panic("isrvec_stk_top overflow %d", stk_top);
vlapic->isrvec_stk[stk_top] = vector;
vlapic_update_ppr(vlapic);
}
void
vlapic_svr_write_handler(struct vlapic *vlapic)
{
struct LAPIC *lapic;
uint32_t old, new, changed;
lapic = vlapic->apic_page;
new = lapic->svr;
old = vlapic->svr_last;
vlapic->svr_last = new;
changed = old ^ new;
if ((changed & APIC_SVR_ENABLE) != 0) {
if ((new & APIC_SVR_ENABLE) == 0) {
/*
* The apic is now disabled so stop the apic timer
* and mask all the LVT entries.
*/
VLAPIC_CTR0(vlapic, "vlapic is software-disabled");
VLAPIC_TIMER_LOCK(vlapic);
callout_stop(&vlapic->callout);
VLAPIC_TIMER_UNLOCK(vlapic);
vlapic_mask_lvts(vlapic);
} else {
/*
* The apic is now enabled so restart the apic timer
* if it is configured in periodic mode.
*/
VLAPIC_CTR0(vlapic, "vlapic is software-enabled");
if (vlapic_periodic_timer(vlapic))
vlapic_icrtmr_write_handler(vlapic);
}
}
}
int
vlapic_read(struct vlapic *vlapic, int mmio_access, uint64_t offset,
uint64_t *data, bool *retu)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *reg;
int i;
/* Ignore MMIO accesses in x2APIC mode */
if (x2apic(vlapic) && mmio_access) {
VLAPIC_CTR1(vlapic, "MMIO read from offset %#lx in x2APIC mode",
offset);
*data = 0;
goto done;
}
if (!x2apic(vlapic) && !mmio_access) {
/*
* XXX Generate GP fault for MSR accesses in xAPIC mode
*/
VLAPIC_CTR1(vlapic, "x2APIC MSR read from offset %#lx in "
"xAPIC mode", offset);
*data = 0;
goto done;
}
if (offset > sizeof(*lapic)) {
*data = 0;
goto done;
}
offset &= ~3;
switch(offset)
{
case APIC_OFFSET_ID:
*data = lapic->id;
break;
case APIC_OFFSET_VER:
*data = lapic->version;
break;
case APIC_OFFSET_TPR:
*data = vlapic_get_tpr(vlapic);
break;
case APIC_OFFSET_APR:
*data = lapic->apr;
break;
case APIC_OFFSET_PPR:
*data = lapic->ppr;
break;
case APIC_OFFSET_EOI:
*data = lapic->eoi;
break;
case APIC_OFFSET_LDR:
*data = lapic->ldr;
break;
case APIC_OFFSET_DFR:
*data = lapic->dfr;
break;
case APIC_OFFSET_SVR:
*data = lapic->svr;
break;
case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
i = (offset - APIC_OFFSET_ISR0) >> 2;
reg = &lapic->isr0;
*data = *(reg + i);
break;
case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
i = (offset - APIC_OFFSET_TMR0) >> 2;
reg = &lapic->tmr0;
*data = *(reg + i);
break;
case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
i = (offset - APIC_OFFSET_IRR0) >> 2;
reg = &lapic->irr0;
*data = atomic_load_acq_int(reg + i);
break;
case APIC_OFFSET_ESR:
*data = lapic->esr;
break;
case APIC_OFFSET_ICR_LOW:
*data = lapic->icr_lo;
if (x2apic(vlapic))
*data |= (uint64_t)lapic->icr_hi << 32;
break;
case APIC_OFFSET_ICR_HI:
*data = lapic->icr_hi;
break;
case APIC_OFFSET_CMCI_LVT:
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
*data = vlapic_get_lvt(vlapic, offset);
#ifdef INVARIANTS
reg = vlapic_get_lvtptr(vlapic, offset);
KASSERT(*data == *reg, ("inconsistent lvt value at "
"offset %#lx: %#lx/%#x", offset, *data, *reg));
#endif
break;
case APIC_OFFSET_TIMER_ICR:
*data = lapic->icr_timer;
break;
case APIC_OFFSET_TIMER_CCR:
*data = vlapic_get_ccr(vlapic);
break;
case APIC_OFFSET_TIMER_DCR:
*data = lapic->dcr_timer;
break;
case APIC_OFFSET_SELF_IPI:
/*
* XXX generate a GP fault if vlapic is in x2apic mode
*/
*data = 0;
break;
case APIC_OFFSET_RRR:
default:
*data = 0;
break;
}
done:
VLAPIC_CTR2(vlapic, "vlapic read offset %#x, data %#lx", offset, *data);
return 0;
}
int
vlapic_write(struct vlapic *vlapic, int mmio_access, uint64_t offset,
uint64_t data, bool *retu)
{
struct LAPIC *lapic = vlapic->apic_page;
uint32_t *regptr;
int retval;
KASSERT((offset & 0xf) == 0 && offset < PAGE_SIZE,
("vlapic_write: invalid offset %#lx", offset));
VLAPIC_CTR2(vlapic, "vlapic write offset %#lx, data %#lx",
offset, data);
if (offset > sizeof(*lapic))
return (0);
/* Ignore MMIO accesses in x2APIC mode */
if (x2apic(vlapic) && mmio_access) {
VLAPIC_CTR2(vlapic, "MMIO write of %#lx to offset %#lx "
"in x2APIC mode", data, offset);
return (0);
}
/*
* XXX Generate GP fault for MSR accesses in xAPIC mode
*/
if (!x2apic(vlapic) && !mmio_access) {
VLAPIC_CTR2(vlapic, "x2APIC MSR write of %#lx to offset %#lx "
"in xAPIC mode", data, offset);
return (0);
}
retval = 0;
switch(offset)
{
case APIC_OFFSET_ID:
lapic->id = data;
vlapic_id_write_handler(vlapic);
break;
case APIC_OFFSET_TPR:
vlapic_set_tpr(vlapic, data & 0xff);
break;
case APIC_OFFSET_EOI:
vlapic_process_eoi(vlapic);
break;
case APIC_OFFSET_LDR:
lapic->ldr = data;
vlapic_ldr_write_handler(vlapic);
break;
case APIC_OFFSET_DFR:
lapic->dfr = data;
vlapic_dfr_write_handler(vlapic);
break;
case APIC_OFFSET_SVR:
lapic->svr = data;
vlapic_svr_write_handler(vlapic);
break;
case APIC_OFFSET_ICR_LOW:
lapic->icr_lo = data;
if (x2apic(vlapic))
lapic->icr_hi = data >> 32;
retval = vlapic_icrlo_write_handler(vlapic, retu);
break;
case APIC_OFFSET_ICR_HI:
lapic->icr_hi = data;
break;
case APIC_OFFSET_CMCI_LVT:
case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT:
regptr = vlapic_get_lvtptr(vlapic, offset);
*regptr = data;
vlapic_lvt_write_handler(vlapic, offset);
break;
case APIC_OFFSET_TIMER_ICR:
lapic->icr_timer = data;
vlapic_icrtmr_write_handler(vlapic);
break;
case APIC_OFFSET_TIMER_DCR:
lapic->dcr_timer = data;
vlapic_dcr_write_handler(vlapic);
break;
case APIC_OFFSET_ESR:
vlapic_esr_write_handler(vlapic);
break;
case APIC_OFFSET_SELF_IPI:
if (x2apic(vlapic))
vlapic_self_ipi_handler(vlapic, data);
break;
case APIC_OFFSET_VER:
case APIC_OFFSET_APR:
case APIC_OFFSET_PPR:
case APIC_OFFSET_RRR:
case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7:
case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7:
case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7:
case APIC_OFFSET_TIMER_CCR:
default:
// Read only.
break;
}
return (retval);
}
static void
vlapic_reset(struct vlapic *vlapic)
{
struct LAPIC *lapic;
lapic = vlapic->apic_page;
bzero(lapic, sizeof(struct LAPIC));
lapic->id = vlapic_get_id(vlapic);
lapic->version = VLAPIC_VERSION;
lapic->version |= (VLAPIC_MAXLVT_INDEX << MAXLVTSHIFT);
lapic->dfr = 0xffffffff;
lapic->svr = APIC_SVR_VECTOR;
vlapic_mask_lvts(vlapic);
vlapic_reset_tmr(vlapic);
lapic->dcr_timer = 0;
vlapic_dcr_write_handler(vlapic);
if (vlapic->vcpuid == 0)
vlapic->boot_state = BS_RUNNING; /* BSP */
else
vlapic->boot_state = BS_INIT; /* AP */
vlapic->svr_last = lapic->svr;
}
void
vlapic_init(struct vlapic *vlapic)
{
KASSERT(vlapic->vm != NULL, ("vlapic_init: vm is not initialized"));
KASSERT(vlapic->vcpuid >= 0 && vlapic->vcpuid < VM_MAXCPU,
("vlapic_init: vcpuid is not initialized"));
KASSERT(vlapic->apic_page != NULL, ("vlapic_init: apic_page is not "
"initialized"));
/*
* If the vlapic is configured in x2apic mode then it will be
* accessed in the critical section via the MSR emulation code.
*
* Therefore the timer mutex must be a spinlock because blockable
* mutexes cannot be acquired in a critical section.
*/
mtx_init(&vlapic->timer_mtx, "vlapic timer mtx", NULL, MTX_SPIN);
callout_init(&vlapic->callout, 1);
vlapic->msr_apicbase = DEFAULT_APIC_BASE | APICBASE_ENABLED;
if (vlapic->vcpuid == 0)
vlapic->msr_apicbase |= APICBASE_BSP;
vlapic_reset(vlapic);
}
void
vlapic_cleanup(struct vlapic *vlapic)
{
callout_drain(&vlapic->callout);
}
uint64_t
vlapic_get_apicbase(struct vlapic *vlapic)
{
return (vlapic->msr_apicbase);
}
int
vlapic_set_apicbase(struct vlapic *vlapic, uint64_t new)
{
if (vlapic->msr_apicbase != new) {
VLAPIC_CTR2(vlapic, "Changing APIC_BASE MSR from %#lx to %#lx "
"not supported", vlapic->msr_apicbase, new);
return (-1);
}
return (0);
}
void
vlapic_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state)
{
struct vlapic *vlapic;
struct LAPIC *lapic;
vlapic = vm_lapic(vm, vcpuid);
if (state == X2APIC_DISABLED)
vlapic->msr_apicbase &= ~APICBASE_X2APIC;
else
vlapic->msr_apicbase |= APICBASE_X2APIC;
/*
* Reset the local APIC registers whose values are mode-dependent.
*
* XXX this works because the APIC mode can be changed only at vcpu
* initialization time.
*/
lapic = vlapic->apic_page;
lapic->id = vlapic_get_id(vlapic);
if (x2apic(vlapic)) {
lapic->ldr = x2apic_ldr(vlapic);
lapic->dfr = 0;
} else {
lapic->ldr = 0;
lapic->dfr = 0xffffffff;
}
if (state == X2APIC_ENABLED) {
if (vlapic->ops.enable_x2apic_mode)
(*vlapic->ops.enable_x2apic_mode)(vlapic);
}
}
void
vlapic_deliver_intr(struct vm *vm, bool level, uint32_t dest, bool phys,
int delmode, int vec)
{
bool lowprio;
int vcpuid;
cpuset_t dmask;
if (delmode != IOART_DELFIXED &&
delmode != IOART_DELLOPRI &&
delmode != IOART_DELEXINT) {
VM_CTR1(vm, "vlapic intr invalid delmode %#x", delmode);
return;
}
lowprio = (delmode == IOART_DELLOPRI);
/*
* We don't provide any virtual interrupt redirection hardware so
* all interrupts originating from the ioapic or MSI specify the
* 'dest' in the legacy xAPIC format.
*/
vlapic_calcdest(vm, &dmask, dest, phys, lowprio, false);
while ((vcpuid = CPU_FFS(&dmask)) != 0) {
vcpuid--;
CPU_CLR(vcpuid, &dmask);
if (delmode == IOART_DELEXINT) {
vm_inject_extint(vm, vcpuid);
} else {
lapic_set_intr(vm, vcpuid, vec, level);
}
}
}
void
vlapic_post_intr(struct vlapic *vlapic, int hostcpu, int ipinum)
{
/*
* Post an interrupt to the vcpu currently running on 'hostcpu'.
*
* This is done by leveraging features like Posted Interrupts (Intel)
* Doorbell MSR (AMD AVIC) that avoid a VM exit.
*
* If neither of these features are available then fallback to
* sending an IPI to 'hostcpu'.
*/
if (vlapic->ops.post_intr)
(*vlapic->ops.post_intr)(vlapic, hostcpu);
else
ipi_cpu(hostcpu, ipinum);
}
bool
vlapic_enabled(struct vlapic *vlapic)
{
struct LAPIC *lapic = vlapic->apic_page;
if ((vlapic->msr_apicbase & APICBASE_ENABLED) != 0 &&
(lapic->svr & APIC_SVR_ENABLE) != 0)
return (true);
else
return (false);
}
static void
vlapic_set_tmr(struct vlapic *vlapic, int vector, bool level)
{
struct LAPIC *lapic;
uint32_t *tmrptr, mask;
int idx;
lapic = vlapic->apic_page;
tmrptr = &lapic->tmr0;
idx = (vector / 32) * 4;
mask = 1 << (vector % 32);
if (level)
tmrptr[idx] |= mask;
else
tmrptr[idx] &= ~mask;
if (vlapic->ops.set_tmr != NULL)
(*vlapic->ops.set_tmr)(vlapic, vector, level);
}
void
vlapic_reset_tmr(struct vlapic *vlapic)
{
int vector;
VLAPIC_CTR0(vlapic, "vlapic resetting all vectors to edge-triggered");
for (vector = 0; vector <= 255; vector++)
vlapic_set_tmr(vlapic, vector, false);
}
void
vlapic_set_tmr_level(struct vlapic *vlapic, uint32_t dest, bool phys,
int delmode, int vector)
{
cpuset_t dmask;
bool lowprio;
KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector));
/*
* A level trigger is valid only for fixed and lowprio delivery modes.
*/
if (delmode != APIC_DELMODE_FIXED && delmode != APIC_DELMODE_LOWPRIO) {
VLAPIC_CTR1(vlapic, "Ignoring level trigger-mode for "
"delivery-mode %d", delmode);
return;
}
lowprio = (delmode == APIC_DELMODE_LOWPRIO);
vlapic_calcdest(vlapic->vm, &dmask, dest, phys, lowprio, false);
if (!CPU_ISSET(vlapic->vcpuid, &dmask))
return;
VLAPIC_CTR1(vlapic, "vector %d set to level-triggered", vector);
vlapic_set_tmr(vlapic, vector, true);
}
Reference in New Issue View Git Blame Copy Permalink