Minor fixes and tweaks to the x86 interrupt code:
- Split the intr_table_lock into an sx lock used for most things, and a spin lock to protect intrcnt_index. Originally I had this as a spin lock so interrupt code could use it to lookup sources. However, we don't actually do that because it would add a lot of overhead to interrupts, and if we ever do support removing interrupt sources, we can use other means to safely do so w/o locking in the interrupt handling code. - Replace is_enabled (boolean) with is_handlers (a count of handlers) to determine if a source is enabled or not. This allows us to notice when a source is no longer in use. When that happens, we now invoke a new PIC method (pic_disable_intr()) to inform the PIC driver that the source is no longer in use. The I/O APIC driver frees the APIC IDT vector when this happens. The MSI driver no longer needs to have a hack to clear is_enabled during msi_alloc() and msix_alloc() as a result of this change as well. - Add an apic_disable_vector() to reset an IDT vector back to Xrsvd to complement apic_enable_vector() and use it in the I/O APIC and MSI code when freeing an IDT vector. - Add a new nexus hook: nexus_add_irq() to ask the nexus driver to add an IRQ to its irq_rman. The MSI code uses this when it creates new interrupt sources to let the nexus know about newly valid IRQs. Previously the msi_alloc() and msix_alloc() passed some extra stuff back to the nexus methods which then added the IRQs. This approach is a bit cleaner. - Change the MSI sx lock to a mutex. If we need to create new sources, drop the lock, create the required number of sources, then get the lock and try the allocation again.
This commit is contained in:
John Baldwin
parent
d287f59062
commit
fb610ca1f9
@ -43,13 +43,14 @@
|
||||
#include <sys/param.h>
|
||||
#include <sys/bus.h>
|
||||
#include <sys/interrupt.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/ktr.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/syslog.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/sx.h>
|
||||
#include <machine/clock.h>
|
||||
#include <machine/intr_machdep.h>
|
||||
#include <machine/smp.h>
|
||||
@ -71,7 +72,8 @@ typedef void (*mask_fn)(void *);
|
||||
|
||||
static int intrcnt_index;
|
||||
static struct intsrc *interrupt_sources[NUM_IO_INTS];
|
||||
static struct mtx intr_table_lock;
|
||||
static struct sx intr_table_lock;
|
||||
static struct mtx intrcnt_lock;
|
||||
static STAILQ_HEAD(, pic) pics;
|
||||
|
||||
#ifdef INTR_FILTER
|
||||
@ -115,14 +117,14 @@ intr_register_pic(struct pic *pic)
|
||||
{
|
||||
int error;
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
if (intr_pic_registered(pic))
|
||||
error = EBUSY;
|
||||
else {
|
||||
STAILQ_INSERT_TAIL(&pics, pic, pics);
|
||||
error = 0;
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
return (error);
|
||||
}
|
||||
|
||||
@ -150,16 +152,16 @@ intr_register_source(struct intsrc *isrc)
|
||||
#endif
|
||||
if (error)
|
||||
return (error);
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
if (interrupt_sources[vector] != NULL) {
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
intr_event_destroy(isrc->is_event);
|
||||
return (EEXIST);
|
||||
}
|
||||
intrcnt_register(isrc);
|
||||
interrupt_sources[vector] = isrc;
|
||||
isrc->is_enabled = 0;
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_handlers = 0;
|
||||
sx_xunlock(&intr_table_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -183,19 +185,18 @@ intr_add_handler(const char *name, int vector, driver_filter_t filter,
|
||||
error = intr_event_add_handler(isrc->is_event, name, filter, handler,
|
||||
arg, intr_priority(flags), flags, cookiep);
|
||||
if (error == 0) {
|
||||
sx_xlock(&intr_table_lock);
|
||||
intrcnt_updatename(isrc);
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
if (!isrc->is_enabled) {
|
||||
isrc->is_enabled = 1;
|
||||
isrc->is_handlers++;
|
||||
if (isrc->is_handlers == 1) {
|
||||
#ifdef SMP
|
||||
if (assign_cpu)
|
||||
intr_assign_next_cpu(isrc);
|
||||
#endif
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_pic->pic_enable_intr(isrc);
|
||||
} else
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_pic->pic_enable_source(isrc);
|
||||
isrc->is_pic->pic_enable_source(isrc);
|
||||
}
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
@ -208,8 +209,16 @@ intr_remove_handler(void *cookie)
|
||||
|
||||
isrc = intr_handler_source(cookie);
|
||||
error = intr_event_remove_handler(cookie);
|
||||
if (error == 0)
|
||||
if (error == 0) {
|
||||
sx_xlock(&intr_table_lock);
|
||||
isrc->is_handlers--;
|
||||
if (isrc->is_handlers == 0) {
|
||||
isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
|
||||
isrc->is_pic->pic_disable_intr(isrc);
|
||||
}
|
||||
intrcnt_updatename(isrc);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
|
||||
@ -391,12 +400,12 @@ intr_resume(void)
|
||||
#ifndef DEV_ATPIC
|
||||
atpic_reset();
|
||||
#endif
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
STAILQ_FOREACH(pic, &pics, pics) {
|
||||
if (pic->pic_resume != NULL)
|
||||
pic->pic_resume(pic);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
|
||||
void
|
||||
@ -404,12 +413,12 @@ intr_suspend(void)
|
||||
{
|
||||
struct pic *pic;
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
STAILQ_FOREACH(pic, &pics, pics) {
|
||||
if (pic->pic_suspend != NULL)
|
||||
pic->pic_suspend(pic);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -432,8 +441,8 @@ intrcnt_register(struct intsrc *is)
|
||||
{
|
||||
char straystr[MAXCOMLEN + 1];
|
||||
|
||||
/* mtx_assert(&intr_table_lock, MA_OWNED); */
|
||||
KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__));
|
||||
mtx_lock_spin(&intrcnt_lock);
|
||||
is->is_index = intrcnt_index;
|
||||
intrcnt_index += 2;
|
||||
snprintf(straystr, MAXCOMLEN + 1, "stray irq%d",
|
||||
@ -442,17 +451,18 @@ intrcnt_register(struct intsrc *is)
|
||||
is->is_count = &intrcnt[is->is_index];
|
||||
intrcnt_setname(straystr, is->is_index + 1);
|
||||
is->is_straycount = &intrcnt[is->is_index + 1];
|
||||
mtx_unlock_spin(&intrcnt_lock);
|
||||
}
|
||||
|
||||
void
|
||||
intrcnt_add(const char *name, u_long **countp)
|
||||
{
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
mtx_lock_spin(&intrcnt_lock);
|
||||
*countp = &intrcnt[intrcnt_index];
|
||||
intrcnt_setname(name, intrcnt_index);
|
||||
intrcnt_index++;
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
mtx_unlock_spin(&intrcnt_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -462,7 +472,8 @@ intr_init(void *dummy __unused)
|
||||
intrcnt_setname("???", 0);
|
||||
intrcnt_index = 1;
|
||||
STAILQ_INIT(&pics);
|
||||
mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN);
|
||||
sx_init(&intr_table_lock, "intr sources");
|
||||
mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
|
||||
}
|
||||
SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL)
|
||||
|
||||
@ -570,14 +581,14 @@ intr_shuffle_irqs(void *arg __unused)
|
||||
return;
|
||||
|
||||
/* Round-robin assign a CPU to each enabled source. */
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
assign_cpu = 1;
|
||||
for (i = 0; i < NUM_IO_INTS; i++) {
|
||||
isrc = interrupt_sources[i];
|
||||
if (isrc != NULL && isrc->is_enabled)
|
||||
if (isrc != NULL && isrc->is_handlers > 0)
|
||||
intr_assign_next_cpu(isrc);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL)
|
||||
#endif
|
||||
|
||||
@ -114,6 +114,7 @@ static void ioapic_enable_source(struct intsrc *isrc);
|
||||
static void ioapic_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void ioapic_eoi_source(struct intsrc *isrc);
|
||||
static void ioapic_enable_intr(struct intsrc *isrc);
|
||||
static void ioapic_disable_intr(struct intsrc *isrc);
|
||||
static int ioapic_vector(struct intsrc *isrc);
|
||||
static int ioapic_source_pending(struct intsrc *isrc);
|
||||
static int ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
@ -125,8 +126,8 @@ static void ioapic_program_intpin(struct ioapic_intsrc *intpin);
|
||||
static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
|
||||
struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
|
||||
ioapic_eoi_source, ioapic_enable_intr,
|
||||
ioapic_vector, ioapic_source_pending,
|
||||
NULL, ioapic_resume,
|
||||
ioapic_disable_intr, ioapic_vector,
|
||||
ioapic_source_pending, NULL, ioapic_resume,
|
||||
ioapic_config_intr, ioapic_assign_cpu };
|
||||
|
||||
static int next_ioapic_base;
|
||||
@ -359,6 +360,23 @@ ioapic_enable_intr(struct intsrc *isrc)
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
ioapic_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
|
||||
u_int vector;
|
||||
|
||||
if (intpin->io_vector != 0) {
|
||||
/* Mask this interrupt pin and free its APIC vector. */
|
||||
vector = intpin->io_vector;
|
||||
apic_disable_vector(vector);
|
||||
intpin->io_masked = 1;
|
||||
intpin->io_vector = 0;
|
||||
ioapic_program_intpin(intpin);
|
||||
apic_free_vector(vector, intpin->io_irq);
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
ioapic_vector(struct intsrc *isrc)
|
||||
{
|
||||
|
||||
@ -147,6 +147,8 @@ static u_int32_t lapic_timer_divisors[] = {
|
||||
APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
|
||||
};
|
||||
|
||||
extern inthand_t IDTVEC(rsvd);
|
||||
|
||||
volatile lapic_t *lapic;
|
||||
vm_paddr_t lapic_paddr;
|
||||
static u_long lapic_timer_divisor, lapic_timer_period, lapic_timer_hz;
|
||||
@ -837,6 +839,16 @@ apic_enable_vector(u_int vector)
|
||||
setidt(vector, ioint_handlers[vector / 32], SDT_SYSIGT, SEL_KPL, 0);
|
||||
}
|
||||
|
||||
void
|
||||
apic_disable_vector(u_int vector)
|
||||
{
|
||||
|
||||
KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
|
||||
KASSERT(ioint_handlers[vector / 32] != NULL,
|
||||
("No ISR handler for vector %u", vector));
|
||||
setidt(vector, &IDTVEC(rsvd), SDT_SYSIGT, SEL_KPL, 0);
|
||||
}
|
||||
|
||||
/* Release an APIC vector when it's no longer in use. */
|
||||
void
|
||||
apic_free_vector(u_int vector, u_int irq)
|
||||
|
||||
@ -112,11 +112,12 @@ struct msi_intsrc {
|
||||
u_int msi_count:8; /* Messages in this group. (g) */
|
||||
};
|
||||
|
||||
static struct msi_intsrc *msi_create_source(u_int irq);
|
||||
static void msi_create_source(void);
|
||||
static void msi_enable_source(struct intsrc *isrc);
|
||||
static void msi_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void msi_eoi_source(struct intsrc *isrc);
|
||||
static void msi_enable_intr(struct intsrc *isrc);
|
||||
static void msi_disable_intr(struct intsrc *isrc);
|
||||
static int msi_vector(struct intsrc *isrc);
|
||||
static int msi_source_pending(struct intsrc *isrc);
|
||||
static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
@ -124,11 +125,13 @@ static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
static void msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
|
||||
|
||||
struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
|
||||
msi_enable_intr, msi_vector, msi_source_pending,
|
||||
NULL, NULL, msi_config_intr, msi_assign_cpu };
|
||||
msi_enable_intr, msi_disable_intr, msi_vector,
|
||||
msi_source_pending, NULL, NULL, msi_config_intr,
|
||||
msi_assign_cpu };
|
||||
|
||||
static int msi_enabled;
|
||||
static struct sx msi_sx;
|
||||
static int msi_last_irq;
|
||||
static struct mtx msi_lock;
|
||||
|
||||
static void
|
||||
msi_enable_source(struct intsrc *isrc)
|
||||
@ -158,6 +161,14 @@ msi_enable_intr(struct intsrc *isrc)
|
||||
apic_enable_vector(msi->msi_vector);
|
||||
}
|
||||
|
||||
static void
|
||||
msi_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
|
||||
|
||||
apic_disable_vector(msi->msi_vector);
|
||||
}
|
||||
|
||||
static int
|
||||
msi_vector(struct intsrc *isrc)
|
||||
{
|
||||
@ -191,8 +202,7 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
|
||||
printf("msi: Assigning %s IRQ %d to local APIC %u\n",
|
||||
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
|
||||
msi->msi_cpu);
|
||||
if (isrc->is_enabled)
|
||||
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
|
||||
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
|
||||
}
|
||||
|
||||
void
|
||||
@ -206,19 +216,29 @@ msi_init(void)
|
||||
|
||||
msi_enabled = 1;
|
||||
intr_register_pic(&msi_pic);
|
||||
sx_init(&msi_sx, "msi");
|
||||
mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
|
||||
}
|
||||
|
||||
struct msi_intsrc *
|
||||
msi_create_source(u_int irq)
|
||||
void
|
||||
msi_create_source(void)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
u_int irq;
|
||||
|
||||
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
|
||||
mtx_lock(&msi_lock);
|
||||
if (msi_last_irq >= NUM_MSI_INTS) {
|
||||
mtx_unlock(&msi_lock);
|
||||
return;
|
||||
}
|
||||
irq = msi_last_irq + FIRST_MSI_INT;
|
||||
msi_last_irq++;
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
|
||||
msi->msi_intsrc.is_pic = &msi_pic;
|
||||
msi->msi_irq = irq;
|
||||
intr_register_source(&msi->msi_intsrc);
|
||||
return (msi);
|
||||
nexus_add_irq(irq);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -228,18 +248,16 @@ msi_create_source(u_int irq)
|
||||
* and *newcount being the number of new IRQ values added.
|
||||
*/
|
||||
int
|
||||
msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
int *newcount)
|
||||
msi_alloc(device_t dev, int count, int maxcount, int *irqs)
|
||||
{
|
||||
struct msi_intsrc *msi, *fsrc;
|
||||
int cnt, i, j, vector;
|
||||
int cnt, i, vector;
|
||||
|
||||
*newirq = 0;
|
||||
*newcount = 0;
|
||||
if (!msi_enabled)
|
||||
return (ENXIO);
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
again:
|
||||
mtx_lock(&msi_lock);
|
||||
|
||||
/* Try to find 'count' free IRQs. */
|
||||
cnt = 0;
|
||||
@ -263,20 +281,17 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
if (cnt < count) {
|
||||
/* If we would exceed the max, give up. */
|
||||
if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
/* We need count - cnt more sources starting at index 'cnt'. */
|
||||
*newirq = cnt;
|
||||
*newcount = count - cnt;
|
||||
for (j = 0; j < *newcount; j++) {
|
||||
|
||||
/* Create a new MSI source and add it to our array. */
|
||||
msi_create_source(i + j);
|
||||
irqs[cnt] = i + j;
|
||||
/* We need count - cnt more sources. */
|
||||
while (cnt < count) {
|
||||
msi_create_source();
|
||||
cnt++;
|
||||
}
|
||||
goto again;
|
||||
}
|
||||
|
||||
/* Ok, we now have the IRQs allocated. */
|
||||
@ -285,7 +300,7 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
/* Allocate 'count' IDT vectors. */
|
||||
vector = apic_alloc_vectors(irqs, count, maxcount);
|
||||
if (vector == 0) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOSPC);
|
||||
}
|
||||
|
||||
@ -299,12 +314,11 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
printf("msi: routing MSI IRQ %d to vector %u\n",
|
||||
msi->msi_irq, msi->msi_vector);
|
||||
msi->msi_first = fsrc;
|
||||
|
||||
/* XXX: Somewhat gross. */
|
||||
msi->msi_intsrc.is_enabled = 0;
|
||||
KASSERT(msi->msi_intsrc.is_handlers == 0,
|
||||
("dead MSI has handlers"));
|
||||
}
|
||||
fsrc->msi_count = count;
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -315,22 +329,22 @@ msi_release(int *irqs, int count)
|
||||
struct msi_intsrc *msi, *first;
|
||||
int i;
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
|
||||
if (first == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this isn't an MSI-X message. */
|
||||
if (first->msi_msix) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
/* Make sure this message is allocated to a group. */
|
||||
if (first->msi_first == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
@ -339,7 +353,7 @@ msi_release(int *irqs, int count)
|
||||
* the entire group.
|
||||
*/
|
||||
if (first->msi_first != first || first->msi_count != count) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
KASSERT(first->msi_dev != NULL, ("unowned group"));
|
||||
@ -362,7 +376,7 @@ msi_release(int *irqs, int count)
|
||||
first->msi_vector = 0;
|
||||
first->msi_count = 0;
|
||||
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -371,27 +385,27 @@ msi_map(int irq, uint64_t *addr, uint32_t *data)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
|
||||
sx_slock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
msi = (struct msi_intsrc *)intr_lookup_source(irq);
|
||||
if (msi == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this message is allocated to a device. */
|
||||
if (msi->msi_dev == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
/*
|
||||
* If this message isn't an MSI-X message, make sure it's part
|
||||
* of a gruop, and switch to the first message in the
|
||||
* of a group, and switch to the first message in the
|
||||
* group.
|
||||
*/
|
||||
if (!msi->msi_msix) {
|
||||
if (msi->msi_first == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
msi = msi->msi_first;
|
||||
@ -399,21 +413,21 @@ msi_map(int irq, uint64_t *addr, uint32_t *data)
|
||||
|
||||
*addr = INTEL_ADDR(msi);
|
||||
*data = INTEL_DATA(msi);
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
msix_alloc(device_t dev, int *irq, int *new)
|
||||
msix_alloc(device_t dev, int *irq)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
int i, vector;
|
||||
|
||||
*new = 0;
|
||||
if (!msi_enabled)
|
||||
return (ENXIO);
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
again:
|
||||
mtx_lock(&msi_lock);
|
||||
|
||||
/* Find a free IRQ. */
|
||||
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
|
||||
@ -423,7 +437,7 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
if (msi == NULL)
|
||||
break;
|
||||
|
||||
/* If this is a free one, start or continue a run. */
|
||||
/* Stop at the first free source. */
|
||||
if (msi->msi_dev == NULL)
|
||||
break;
|
||||
}
|
||||
@ -432,13 +446,14 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
if (msi == NULL) {
|
||||
/* If we would exceed the max, give up. */
|
||||
if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
/* Create a new source. */
|
||||
*new = 1;
|
||||
msi = msi_create_source(i);
|
||||
msi_create_source();
|
||||
goto again;
|
||||
}
|
||||
|
||||
/* Allocate an IDT vector. */
|
||||
@ -452,9 +467,8 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
msi->msi_vector = vector;
|
||||
msi->msi_msix = 1;
|
||||
|
||||
/* XXX: Somewhat gross. */
|
||||
msi->msi_intsrc.is_enabled = 0;
|
||||
sx_xunlock(&msi_sx);
|
||||
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
*irq = i;
|
||||
return (0);
|
||||
@ -465,16 +479,16 @@ msix_release(int irq)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
msi = (struct msi_intsrc *)intr_lookup_source(irq);
|
||||
if (msi == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this is an MSI-X message. */
|
||||
if (!msi->msi_msix) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
@ -486,6 +500,6 @@ msix_release(int irq)
|
||||
msi->msi_vector = 0;
|
||||
msi->msi_msix = 0;
|
||||
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -503,15 +503,20 @@ nexus_delete_resource(device_t dev, device_t child, int type, int rid)
|
||||
resource_list_delete(rl, type, rid);
|
||||
}
|
||||
|
||||
/* Called from the MSI code to add new IRQs to the IRQ rman. */
|
||||
void
|
||||
nexus_add_irq(u_long irq)
|
||||
{
|
||||
|
||||
if (rman_manage_region(&irq_rman, irq, irq) != 0)
|
||||
panic("%s: failed", __func__);
|
||||
}
|
||||
|
||||
static int
|
||||
nexus_alloc_msix(device_t pcib, device_t dev, int *irq)
|
||||
{
|
||||
int error, new;
|
||||
|
||||
error = msix_alloc(dev, irq, &new);
|
||||
if (new)
|
||||
rman_manage_region(&irq_rman, *irq, *irq);
|
||||
return (error);
|
||||
return (msix_alloc(dev, irq));
|
||||
}
|
||||
|
||||
static int
|
||||
@ -524,17 +529,8 @@ nexus_release_msix(device_t pcib, device_t dev, int irq)
|
||||
static int
|
||||
nexus_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs)
|
||||
{
|
||||
int error, i, newirq, newcount;
|
||||
|
||||
/* First alloc the messages. */
|
||||
error = msi_alloc(dev, count, maxcount, irqs, &newirq, &newcount);
|
||||
|
||||
/* Always add any new IRQs to the rman, even on failure. */
|
||||
for (i = 0; i < newcount; i++)
|
||||
rman_manage_region(&irq_rman, irqs[newirq + i],
|
||||
irqs[newirq + i]);
|
||||
|
||||
return (error);
|
||||
return (msi_alloc(dev, count, maxcount, irqs));
|
||||
}
|
||||
|
||||
static int
|
||||
|
||||
@ -178,6 +178,7 @@ extern vm_paddr_t lapic_paddr;
|
||||
|
||||
u_int apic_alloc_vector(u_int irq);
|
||||
u_int apic_alloc_vectors(u_int *irqs, u_int count, u_int align);
|
||||
void apic_disable_vector(u_int vector);
|
||||
void apic_enable_vector(u_int vector);
|
||||
void apic_free_vector(u_int vector, u_int irq);
|
||||
u_int apic_idt_to_irq(u_int vector);
|
||||
|
||||
@ -86,6 +86,7 @@ struct pic {
|
||||
void (*pic_disable_source)(struct intsrc *, int);
|
||||
void (*pic_eoi_source)(struct intsrc *);
|
||||
void (*pic_enable_intr)(struct intsrc *);
|
||||
void (*pic_disable_intr)(struct intsrc *);
|
||||
int (*pic_vector)(struct intsrc *);
|
||||
int (*pic_source_pending)(struct intsrc *);
|
||||
void (*pic_suspend)(struct pic *);
|
||||
@ -114,7 +115,7 @@ struct intsrc {
|
||||
u_long *is_count;
|
||||
u_long *is_straycount;
|
||||
u_int is_index;
|
||||
u_int is_enabled:1;
|
||||
u_int is_handlers;
|
||||
};
|
||||
|
||||
struct trapframe;
|
||||
@ -146,12 +147,12 @@ int intr_remove_handler(void *cookie);
|
||||
void intr_resume(void);
|
||||
void intr_suspend(void);
|
||||
void intrcnt_add(const char *name, u_long **countp);
|
||||
int msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
int *newcount);
|
||||
void nexus_add_irq(u_long irq);
|
||||
int msi_alloc(device_t dev, int count, int maxcount, int *irqs);
|
||||
void msi_init(void);
|
||||
int msi_map(int irq, uint64_t *addr, uint32_t *data);
|
||||
int msi_release(int *irqs, int count);
|
||||
int msix_alloc(device_t dev, int *irq, int *new);
|
||||
int msix_alloc(device_t dev, int *irq);
|
||||
int msix_release(int irq);
|
||||
|
||||
#endif /* !LOCORE */
|
||||
|
||||
@ -107,9 +107,10 @@ inthand_t
|
||||
|
||||
#define ATPIC(io, base, eoi, imenptr) \
|
||||
{ { atpic_enable_source, atpic_disable_source, (eoi), \
|
||||
atpic_enable_intr, atpic_vector, atpic_source_pending, NULL, \
|
||||
atpic_resume, atpic_config_intr, atpic_assign_cpu }, (io), \
|
||||
(base), IDT_IO_INTS + (base), (imenptr) }
|
||||
atpic_enable_intr, atpic_disable_intr, atpic_vector, \
|
||||
atpic_source_pending, NULL, atpic_resume, atpic_config_intr,\
|
||||
atpic_assign_cpu }, (io), (base), IDT_IO_INTS + (base), \
|
||||
(imenptr) }
|
||||
|
||||
#define INTSRC(irq) \
|
||||
{ { &atpics[(irq) / 8].at_pic }, IDTVEC(atpic_intr ## irq ), \
|
||||
@ -137,6 +138,7 @@ static void atpic_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void atpic_eoi_master(struct intsrc *isrc);
|
||||
static void atpic_eoi_slave(struct intsrc *isrc);
|
||||
static void atpic_enable_intr(struct intsrc *isrc);
|
||||
static void atpic_disable_intr(struct intsrc *isrc);
|
||||
static int atpic_vector(struct intsrc *isrc);
|
||||
static void atpic_resume(struct pic *pic);
|
||||
static int atpic_source_pending(struct intsrc *isrc);
|
||||
@ -266,6 +268,12 @@ atpic_enable_intr(struct intsrc *isrc)
|
||||
{
|
||||
}
|
||||
|
||||
static void
|
||||
atpic_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
static int
|
||||
atpic_vector(struct intsrc *isrc)
|
||||
{
|
||||
|
||||
@ -42,13 +42,14 @@
|
||||
#include <sys/param.h>
|
||||
#include <sys/bus.h>
|
||||
#include <sys/interrupt.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/ktr.h>
|
||||
#include <sys/kernel.h>
|
||||
#include <sys/lock.h>
|
||||
#include <sys/mutex.h>
|
||||
#include <sys/proc.h>
|
||||
#include <sys/syslog.h>
|
||||
#include <sys/systm.h>
|
||||
#include <sys/sx.h>
|
||||
#include <machine/clock.h>
|
||||
#include <machine/intr_machdep.h>
|
||||
#include <machine/smp.h>
|
||||
@ -62,7 +63,8 @@ typedef void (*mask_fn)(void *);
|
||||
|
||||
static int intrcnt_index;
|
||||
static struct intsrc *interrupt_sources[NUM_IO_INTS];
|
||||
static struct mtx intr_table_lock;
|
||||
static struct sx intr_table_lock;
|
||||
static struct mtx intrcnt_lock;
|
||||
static STAILQ_HEAD(, pic) pics;
|
||||
|
||||
#ifdef INTR_FILTER
|
||||
@ -106,14 +108,14 @@ intr_register_pic(struct pic *pic)
|
||||
{
|
||||
int error;
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
if (intr_pic_registered(pic))
|
||||
error = EBUSY;
|
||||
else {
|
||||
STAILQ_INSERT_TAIL(&pics, pic, pics);
|
||||
error = 0;
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
return (error);
|
||||
}
|
||||
|
||||
@ -141,16 +143,16 @@ intr_register_source(struct intsrc *isrc)
|
||||
#endif
|
||||
if (error)
|
||||
return (error);
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
if (interrupt_sources[vector] != NULL) {
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
intr_event_destroy(isrc->is_event);
|
||||
return (EEXIST);
|
||||
}
|
||||
intrcnt_register(isrc);
|
||||
interrupt_sources[vector] = isrc;
|
||||
isrc->is_enabled = 0;
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_handlers = 0;
|
||||
sx_xunlock(&intr_table_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -174,19 +176,18 @@ intr_add_handler(const char *name, int vector, driver_filter_t filter,
|
||||
error = intr_event_add_handler(isrc->is_event, name, filter, handler,
|
||||
arg, intr_priority(flags), flags, cookiep);
|
||||
if (error == 0) {
|
||||
sx_xlock(&intr_table_lock);
|
||||
intrcnt_updatename(isrc);
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
if (!isrc->is_enabled) {
|
||||
isrc->is_enabled = 1;
|
||||
isrc->is_handlers++;
|
||||
if (isrc->is_handlers == 1) {
|
||||
#ifdef SMP
|
||||
if (assign_cpu)
|
||||
intr_assign_next_cpu(isrc);
|
||||
#endif
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_pic->pic_enable_intr(isrc);
|
||||
} else
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
isrc->is_pic->pic_enable_source(isrc);
|
||||
isrc->is_pic->pic_enable_source(isrc);
|
||||
}
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
@ -199,8 +200,16 @@ intr_remove_handler(void *cookie)
|
||||
|
||||
isrc = intr_handler_source(cookie);
|
||||
error = intr_event_remove_handler(cookie);
|
||||
if (error == 0)
|
||||
if (error == 0) {
|
||||
sx_xlock(&intr_table_lock);
|
||||
isrc->is_handlers--;
|
||||
if (isrc->is_handlers == 0) {
|
||||
isrc->is_pic->pic_disable_source(isrc, PIC_NO_EOI);
|
||||
isrc->is_pic->pic_disable_intr(isrc);
|
||||
}
|
||||
intrcnt_updatename(isrc);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
return (error);
|
||||
}
|
||||
|
||||
@ -379,12 +388,12 @@ intr_resume(void)
|
||||
{
|
||||
struct pic *pic;
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
STAILQ_FOREACH(pic, &pics, pics) {
|
||||
if (pic->pic_resume != NULL)
|
||||
pic->pic_resume(pic);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
|
||||
void
|
||||
@ -392,12 +401,12 @@ intr_suspend(void)
|
||||
{
|
||||
struct pic *pic;
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
STAILQ_FOREACH(pic, &pics, pics) {
|
||||
if (pic->pic_suspend != NULL)
|
||||
pic->pic_suspend(pic);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -420,8 +429,8 @@ intrcnt_register(struct intsrc *is)
|
||||
{
|
||||
char straystr[MAXCOMLEN + 1];
|
||||
|
||||
/* mtx_assert(&intr_table_lock, MA_OWNED); */
|
||||
KASSERT(is->is_event != NULL, ("%s: isrc with no event", __func__));
|
||||
mtx_lock_spin(&intrcnt_lock);
|
||||
is->is_index = intrcnt_index;
|
||||
intrcnt_index += 2;
|
||||
snprintf(straystr, MAXCOMLEN + 1, "stray irq%d",
|
||||
@ -430,17 +439,18 @@ intrcnt_register(struct intsrc *is)
|
||||
is->is_count = &intrcnt[is->is_index];
|
||||
intrcnt_setname(straystr, is->is_index + 1);
|
||||
is->is_straycount = &intrcnt[is->is_index + 1];
|
||||
mtx_unlock_spin(&intrcnt_lock);
|
||||
}
|
||||
|
||||
void
|
||||
intrcnt_add(const char *name, u_long **countp)
|
||||
{
|
||||
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
mtx_lock_spin(&intrcnt_lock);
|
||||
*countp = &intrcnt[intrcnt_index];
|
||||
intrcnt_setname(name, intrcnt_index);
|
||||
intrcnt_index++;
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
mtx_unlock_spin(&intrcnt_lock);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -450,7 +460,8 @@ intr_init(void *dummy __unused)
|
||||
intrcnt_setname("???", 0);
|
||||
intrcnt_index = 1;
|
||||
STAILQ_INIT(&pics);
|
||||
mtx_init(&intr_table_lock, "intr table", NULL, MTX_SPIN);
|
||||
sx_init(&intr_table_lock, "intr sources");
|
||||
mtx_init(&intrcnt_lock, "intrcnt", NULL, MTX_SPIN);
|
||||
}
|
||||
SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL)
|
||||
|
||||
@ -536,14 +547,14 @@ intr_shuffle_irqs(void *arg __unused)
|
||||
return;
|
||||
|
||||
/* Round-robin assign a CPU to each enabled source. */
|
||||
mtx_lock_spin(&intr_table_lock);
|
||||
sx_xlock(&intr_table_lock);
|
||||
assign_cpu = 1;
|
||||
for (i = 0; i < NUM_IO_INTS; i++) {
|
||||
isrc = interrupt_sources[i];
|
||||
if (isrc != NULL && isrc->is_enabled)
|
||||
if (isrc != NULL && isrc->is_handlers > 0)
|
||||
intr_assign_next_cpu(isrc);
|
||||
}
|
||||
mtx_unlock_spin(&intr_table_lock);
|
||||
sx_xunlock(&intr_table_lock);
|
||||
}
|
||||
SYSINIT(intr_shuffle_irqs, SI_SUB_SMP, SI_ORDER_SECOND, intr_shuffle_irqs, NULL)
|
||||
#endif
|
||||
|
||||
@ -114,6 +114,7 @@ static void ioapic_enable_source(struct intsrc *isrc);
|
||||
static void ioapic_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void ioapic_eoi_source(struct intsrc *isrc);
|
||||
static void ioapic_enable_intr(struct intsrc *isrc);
|
||||
static void ioapic_disable_intr(struct intsrc *isrc);
|
||||
static int ioapic_vector(struct intsrc *isrc);
|
||||
static int ioapic_source_pending(struct intsrc *isrc);
|
||||
static int ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
@ -125,8 +126,8 @@ static void ioapic_program_intpin(struct ioapic_intsrc *intpin);
|
||||
static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list);
|
||||
struct pic ioapic_template = { ioapic_enable_source, ioapic_disable_source,
|
||||
ioapic_eoi_source, ioapic_enable_intr,
|
||||
ioapic_vector, ioapic_source_pending,
|
||||
NULL, ioapic_resume,
|
||||
ioapic_disable_intr, ioapic_vector,
|
||||
ioapic_source_pending, NULL, ioapic_resume,
|
||||
ioapic_config_intr, ioapic_assign_cpu };
|
||||
|
||||
static int next_ioapic_base;
|
||||
@ -359,6 +360,23 @@ ioapic_enable_intr(struct intsrc *isrc)
|
||||
}
|
||||
}
|
||||
|
||||
static void
|
||||
ioapic_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc;
|
||||
u_int vector;
|
||||
|
||||
if (intpin->io_vector != 0) {
|
||||
/* Mask this interrupt pin and free its APIC vector. */
|
||||
vector = intpin->io_vector;
|
||||
apic_disable_vector(vector);
|
||||
intpin->io_masked = 1;
|
||||
intpin->io_vector = 0;
|
||||
ioapic_program_intpin(intpin);
|
||||
apic_free_vector(vector, intpin->io_irq);
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
ioapic_vector(struct intsrc *isrc)
|
||||
{
|
||||
|
||||
@ -147,6 +147,8 @@ static u_int32_t lapic_timer_divisors[] = {
|
||||
APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128
|
||||
};
|
||||
|
||||
extern inthand_t IDTVEC(rsvd);
|
||||
|
||||
volatile lapic_t *lapic;
|
||||
vm_paddr_t lapic_paddr;
|
||||
static u_long lapic_timer_divisor, lapic_timer_period, lapic_timer_hz;
|
||||
@ -840,6 +842,17 @@ apic_enable_vector(u_int vector)
|
||||
GSEL(GCODE_SEL, SEL_KPL));
|
||||
}
|
||||
|
||||
void
|
||||
apic_disable_vector(u_int vector)
|
||||
{
|
||||
|
||||
KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry"));
|
||||
KASSERT(ioint_handlers[vector / 32] != NULL,
|
||||
("No ISR handler for vector %u", vector));
|
||||
setidt(vector, &IDTVEC(rsvd), SDT_SYS386TGT, SEL_KPL,
|
||||
GSEL(GCODE_SEL, SEL_KPL));
|
||||
}
|
||||
|
||||
/* Release an APIC vector when it's no longer in use. */
|
||||
void
|
||||
apic_free_vector(u_int vector, u_int irq)
|
||||
|
||||
@ -112,11 +112,12 @@ struct msi_intsrc {
|
||||
u_int msi_count:8; /* Messages in this group. (g) */
|
||||
};
|
||||
|
||||
static struct msi_intsrc *msi_create_source(u_int irq);
|
||||
static void msi_create_source(void);
|
||||
static void msi_enable_source(struct intsrc *isrc);
|
||||
static void msi_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void msi_eoi_source(struct intsrc *isrc);
|
||||
static void msi_enable_intr(struct intsrc *isrc);
|
||||
static void msi_disable_intr(struct intsrc *isrc);
|
||||
static int msi_vector(struct intsrc *isrc);
|
||||
static int msi_source_pending(struct intsrc *isrc);
|
||||
static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
@ -124,11 +125,13 @@ static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
|
||||
static void msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
|
||||
|
||||
struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
|
||||
msi_enable_intr, msi_vector, msi_source_pending,
|
||||
NULL, NULL, msi_config_intr, msi_assign_cpu };
|
||||
msi_enable_intr, msi_disable_intr, msi_vector,
|
||||
msi_source_pending, NULL, NULL, msi_config_intr,
|
||||
msi_assign_cpu };
|
||||
|
||||
static int msi_enabled;
|
||||
static struct sx msi_sx;
|
||||
static int msi_last_irq;
|
||||
static struct mtx msi_lock;
|
||||
|
||||
static void
|
||||
msi_enable_source(struct intsrc *isrc)
|
||||
@ -158,6 +161,14 @@ msi_enable_intr(struct intsrc *isrc)
|
||||
apic_enable_vector(msi->msi_vector);
|
||||
}
|
||||
|
||||
static void
|
||||
msi_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
|
||||
|
||||
apic_disable_vector(msi->msi_vector);
|
||||
}
|
||||
|
||||
static int
|
||||
msi_vector(struct intsrc *isrc)
|
||||
{
|
||||
@ -191,8 +202,7 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
|
||||
printf("msi: Assigning %s IRQ %d to local APIC %u\n",
|
||||
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
|
||||
msi->msi_cpu);
|
||||
if (isrc->is_enabled)
|
||||
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
|
||||
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
|
||||
}
|
||||
|
||||
void
|
||||
@ -206,19 +216,29 @@ msi_init(void)
|
||||
|
||||
msi_enabled = 1;
|
||||
intr_register_pic(&msi_pic);
|
||||
sx_init(&msi_sx, "msi");
|
||||
mtx_init(&msi_lock, "msi", NULL, MTX_DEF);
|
||||
}
|
||||
|
||||
struct msi_intsrc *
|
||||
msi_create_source(u_int irq)
|
||||
void
|
||||
msi_create_source(void)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
u_int irq;
|
||||
|
||||
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
|
||||
mtx_lock(&msi_lock);
|
||||
if (msi_last_irq >= NUM_MSI_INTS) {
|
||||
mtx_unlock(&msi_lock);
|
||||
return;
|
||||
}
|
||||
irq = msi_last_irq + FIRST_MSI_INT;
|
||||
msi_last_irq++;
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
|
||||
msi->msi_intsrc.is_pic = &msi_pic;
|
||||
msi->msi_irq = irq;
|
||||
intr_register_source(&msi->msi_intsrc);
|
||||
return (msi);
|
||||
nexus_add_irq(irq);
|
||||
}
|
||||
|
||||
/*
|
||||
@ -228,18 +248,16 @@ msi_create_source(u_int irq)
|
||||
* and *newcount being the number of new IRQ values added.
|
||||
*/
|
||||
int
|
||||
msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
int *newcount)
|
||||
msi_alloc(device_t dev, int count, int maxcount, int *irqs)
|
||||
{
|
||||
struct msi_intsrc *msi, *fsrc;
|
||||
int cnt, i, j, vector;
|
||||
int cnt, i, vector;
|
||||
|
||||
*newirq = 0;
|
||||
*newcount = 0;
|
||||
if (!msi_enabled)
|
||||
return (ENXIO);
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
again:
|
||||
mtx_lock(&msi_lock);
|
||||
|
||||
/* Try to find 'count' free IRQs. */
|
||||
cnt = 0;
|
||||
@ -263,20 +281,17 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
if (cnt < count) {
|
||||
/* If we would exceed the max, give up. */
|
||||
if (i + (count - cnt) > FIRST_MSI_INT + NUM_MSI_INTS) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
/* We need count - cnt more sources starting at index 'cnt'. */
|
||||
*newirq = cnt;
|
||||
*newcount = count - cnt;
|
||||
for (j = 0; j < *newcount; j++) {
|
||||
|
||||
/* Create a new MSI source and add it to our array. */
|
||||
msi_create_source(i + j);
|
||||
irqs[cnt] = i + j;
|
||||
/* We need count - cnt more sources. */
|
||||
while (cnt < count) {
|
||||
msi_create_source();
|
||||
cnt++;
|
||||
}
|
||||
goto again;
|
||||
}
|
||||
|
||||
/* Ok, we now have the IRQs allocated. */
|
||||
@ -285,7 +300,7 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
/* Allocate 'count' IDT vectors. */
|
||||
vector = apic_alloc_vectors(irqs, count, maxcount);
|
||||
if (vector == 0) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOSPC);
|
||||
}
|
||||
|
||||
@ -299,12 +314,11 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
printf("msi: routing MSI IRQ %d to vector %u\n",
|
||||
msi->msi_irq, msi->msi_vector);
|
||||
msi->msi_first = fsrc;
|
||||
|
||||
/* XXX: Somewhat gross. */
|
||||
msi->msi_intsrc.is_enabled = 0;
|
||||
KASSERT(msi->msi_intsrc.is_handlers == 0,
|
||||
("dead MSI has handlers"));
|
||||
}
|
||||
fsrc->msi_count = count;
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
return (0);
|
||||
}
|
||||
@ -315,22 +329,22 @@ msi_release(int *irqs, int count)
|
||||
struct msi_intsrc *msi, *first;
|
||||
int i;
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
first = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
|
||||
if (first == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this isn't an MSI-X message. */
|
||||
if (first->msi_msix) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
/* Make sure this message is allocated to a group. */
|
||||
if (first->msi_first == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
@ -339,7 +353,7 @@ msi_release(int *irqs, int count)
|
||||
* the entire group.
|
||||
*/
|
||||
if (first->msi_first != first || first->msi_count != count) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
KASSERT(first->msi_dev != NULL, ("unowned group"));
|
||||
@ -362,7 +376,7 @@ msi_release(int *irqs, int count)
|
||||
first->msi_vector = 0;
|
||||
first->msi_count = 0;
|
||||
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -371,27 +385,27 @@ msi_map(int irq, uint64_t *addr, uint32_t *data)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
|
||||
sx_slock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
msi = (struct msi_intsrc *)intr_lookup_source(irq);
|
||||
if (msi == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this message is allocated to a device. */
|
||||
if (msi->msi_dev == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
/*
|
||||
* If this message isn't an MSI-X message, make sure it's part
|
||||
* of a gruop, and switch to the first message in the
|
||||
* of a group, and switch to the first message in the
|
||||
* group.
|
||||
*/
|
||||
if (!msi->msi_msix) {
|
||||
if (msi->msi_first == NULL) {
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
msi = msi->msi_first;
|
||||
@ -399,21 +413,21 @@ msi_map(int irq, uint64_t *addr, uint32_t *data)
|
||||
|
||||
*addr = INTEL_ADDR(msi);
|
||||
*data = INTEL_DATA(msi);
|
||||
sx_sunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
int
|
||||
msix_alloc(device_t dev, int *irq, int *new)
|
||||
msix_alloc(device_t dev, int *irq)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
int i, vector;
|
||||
|
||||
*new = 0;
|
||||
if (!msi_enabled)
|
||||
return (ENXIO);
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
again:
|
||||
mtx_lock(&msi_lock);
|
||||
|
||||
/* Find a free IRQ. */
|
||||
for (i = FIRST_MSI_INT; i < FIRST_MSI_INT + NUM_MSI_INTS; i++) {
|
||||
@ -423,7 +437,7 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
if (msi == NULL)
|
||||
break;
|
||||
|
||||
/* If this is a free one, start or continue a run. */
|
||||
/* Stop at the first free source. */
|
||||
if (msi->msi_dev == NULL)
|
||||
break;
|
||||
}
|
||||
@ -432,13 +446,14 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
if (msi == NULL) {
|
||||
/* If we would exceed the max, give up. */
|
||||
if (i + 1 > FIRST_MSI_INT + NUM_MSI_INTS) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENXIO);
|
||||
}
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
/* Create a new source. */
|
||||
*new = 1;
|
||||
msi = msi_create_source(i);
|
||||
msi_create_source();
|
||||
goto again;
|
||||
}
|
||||
|
||||
/* Allocate an IDT vector. */
|
||||
@ -452,9 +467,8 @@ msix_alloc(device_t dev, int *irq, int *new)
|
||||
msi->msi_vector = vector;
|
||||
msi->msi_msix = 1;
|
||||
|
||||
/* XXX: Somewhat gross. */
|
||||
msi->msi_intsrc.is_enabled = 0;
|
||||
sx_xunlock(&msi_sx);
|
||||
KASSERT(msi->msi_intsrc.is_handlers == 0, ("dead MSI-X has handlers"));
|
||||
mtx_unlock(&msi_lock);
|
||||
|
||||
*irq = i;
|
||||
return (0);
|
||||
@ -465,16 +479,16 @@ msix_release(int irq)
|
||||
{
|
||||
struct msi_intsrc *msi;
|
||||
|
||||
sx_xlock(&msi_sx);
|
||||
mtx_lock(&msi_lock);
|
||||
msi = (struct msi_intsrc *)intr_lookup_source(irq);
|
||||
if (msi == NULL) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (ENOENT);
|
||||
}
|
||||
|
||||
/* Make sure this is an MSI-X message. */
|
||||
if (!msi->msi_msix) {
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (EINVAL);
|
||||
}
|
||||
|
||||
@ -486,6 +500,6 @@ msix_release(int irq)
|
||||
msi->msi_vector = 0;
|
||||
msi->msi_msix = 0;
|
||||
|
||||
sx_xunlock(&msi_sx);
|
||||
mtx_unlock(&msi_lock);
|
||||
return (0);
|
||||
}
|
||||
|
||||
@ -550,16 +550,21 @@ nexus_delete_resource(device_t dev, device_t child, int type, int rid)
|
||||
resource_list_delete(rl, type, rid);
|
||||
}
|
||||
|
||||
/* Called from the MSI code to add new IRQs to the IRQ rman. */
|
||||
void
|
||||
nexus_add_irq(u_long irq)
|
||||
{
|
||||
|
||||
if (rman_manage_region(&irq_rman, irq, irq) != 0)
|
||||
panic("%s: failed", __func__);
|
||||
}
|
||||
|
||||
#ifdef DEV_APIC
|
||||
static int
|
||||
nexus_alloc_msix(device_t pcib, device_t dev, int *irq)
|
||||
{
|
||||
int error, new;
|
||||
|
||||
error = msix_alloc(dev, irq, &new);
|
||||
if (new)
|
||||
rman_manage_region(&irq_rman, *irq, *irq);
|
||||
return (error);
|
||||
return (msix_alloc(dev, irq));
|
||||
}
|
||||
|
||||
static int
|
||||
@ -572,17 +577,8 @@ nexus_release_msix(device_t pcib, device_t dev, int irq)
|
||||
static int
|
||||
nexus_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs)
|
||||
{
|
||||
int error, i, newirq, newcount;
|
||||
|
||||
/* First alloc the messages. */
|
||||
error = msi_alloc(dev, count, maxcount, irqs, &newirq, &newcount);
|
||||
|
||||
/* Always add any new IRQs to the rman, even on failure. */
|
||||
for (i = 0; i < newcount; i++)
|
||||
rman_manage_region(&irq_rman, irqs[newirq + i],
|
||||
irqs[newirq + i]);
|
||||
|
||||
return (error);
|
||||
return (msi_alloc(dev, count, maxcount, irqs));
|
||||
}
|
||||
|
||||
static int
|
||||
|
||||
@ -177,6 +177,7 @@ extern vm_paddr_t lapic_paddr;
|
||||
|
||||
u_int apic_alloc_vector(u_int irq);
|
||||
u_int apic_alloc_vectors(u_int *irqs, u_int count, u_int align);
|
||||
void apic_disable_vector(u_int vector);
|
||||
void apic_enable_vector(u_int vector);
|
||||
void apic_free_vector(u_int vector, u_int irq);
|
||||
u_int apic_idt_to_irq(u_int vector);
|
||||
|
||||
@ -86,6 +86,7 @@ struct pic {
|
||||
void (*pic_disable_source)(struct intsrc *, int);
|
||||
void (*pic_eoi_source)(struct intsrc *);
|
||||
void (*pic_enable_intr)(struct intsrc *);
|
||||
void (*pic_disable_intr)(struct intsrc *);
|
||||
int (*pic_vector)(struct intsrc *);
|
||||
int (*pic_source_pending)(struct intsrc *);
|
||||
void (*pic_suspend)(struct pic *);
|
||||
@ -114,7 +115,7 @@ struct intsrc {
|
||||
u_long *is_count;
|
||||
u_long *is_straycount;
|
||||
u_int is_index;
|
||||
u_int is_enabled:1;
|
||||
u_int is_handlers;
|
||||
};
|
||||
|
||||
struct trapframe;
|
||||
@ -142,12 +143,12 @@ int intr_remove_handler(void *cookie);
|
||||
void intr_resume(void);
|
||||
void intr_suspend(void);
|
||||
void intrcnt_add(const char *name, u_long **countp);
|
||||
int msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
|
||||
int *newcount);
|
||||
void nexus_add_irq(u_long irq);
|
||||
int msi_alloc(device_t dev, int count, int maxcount, int *irqs);
|
||||
void msi_init(void);
|
||||
int msi_map(int irq, uint64_t *addr, uint32_t *data);
|
||||
int msi_release(int* irqs, int count);
|
||||
int msix_alloc(device_t dev, int *irq, int *new);
|
||||
int msix_alloc(device_t dev, int *irq);
|
||||
int msix_release(int irq);
|
||||
|
||||
#endif /* !LOCORE */
|
||||
|
||||
@ -125,9 +125,10 @@ inthand_t
|
||||
|
||||
#define ATPIC(io, base, eoi, imenptr) \
|
||||
{ { atpic_enable_source, atpic_disable_source, (eoi), \
|
||||
atpic_enable_intr, atpic_vector, atpic_source_pending, NULL, \
|
||||
atpic_resume, atpic_config_intr, atpic_assign_cpu }, (io), \
|
||||
(base), IDT_IO_INTS + (base), (imenptr) }
|
||||
atpic_enable_intr, atpic_disable_intr, atpic_vector, \
|
||||
atpic_source_pending, NULL, atpic_resume, atpic_config_intr,\
|
||||
atpic_assign_cpu }, (io), (base), IDT_IO_INTS + (base), \
|
||||
(imenptr) }
|
||||
|
||||
#define INTSRC(irq) \
|
||||
{ { &atpics[(irq) / 8].at_pic }, IDTVEC(atpic_intr ## irq ), \
|
||||
@ -155,6 +156,7 @@ static void atpic_disable_source(struct intsrc *isrc, int eoi);
|
||||
static void atpic_eoi_master(struct intsrc *isrc);
|
||||
static void atpic_eoi_slave(struct intsrc *isrc);
|
||||
static void atpic_enable_intr(struct intsrc *isrc);
|
||||
static void atpic_disable_intr(struct intsrc *isrc);
|
||||
static int atpic_vector(struct intsrc *isrc);
|
||||
static void atpic_resume(struct pic *pic);
|
||||
static int atpic_source_pending(struct intsrc *isrc);
|
||||
@ -284,6 +286,12 @@ atpic_enable_intr(struct intsrc *isrc)
|
||||
{
|
||||
}
|
||||
|
||||
static void
|
||||
atpic_disable_intr(struct intsrc *isrc)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
static int
|
||||
atpic_vector(struct intsrc *isrc)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user