Rework the interrupt code and add support for interrupt filtering

(INTR_FILTER). This includes:
o  Save a pointer to the sapic structure and IRQ for every vector,
   so that we can quickly EOI, mask and unmask the interrupt.
o  Add locking to the sapic code now that we can reprogram a
   sapic on multiple CPUs at the same time.
o  Use u_int for the vector and IRQ. We only have 256 vectors, so
   using a 64-bit type for it is rather excessive.
o  Properly handle concurrent registration of a handler for the
   same vector.

Since vectors have a corresponding priority, we should not map
IRQs to vectors in a linear fashion, but rather pick a vector
that has a priority in line with the interrupt type. This is left
for later. The vector/IRQ interchange has been untangled as much
as possible to make this easier.

Approved by: re (blacket)
This commit is contained in:
Marcel Moolenaar 2007-07-30 22:29:33 +00:00
parent 8a2a70cb02
commit fe1c66b9d7
Notes: svn2git 2020-12-20 02:59:44 +00:00
svn path=/head/; revision=171664
6 changed files with 238 additions and 176 deletions

View File

@ -49,6 +49,7 @@
#include <sys/mutex.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
#include <sys/syslog.h>
#include <machine/clock.h>
#include <machine/cpu.h>
@ -76,7 +77,7 @@ struct evcnt clock_intr_evcnt; /* event counter for clock intrs. */
extern int mp_ipi_test;
#endif
volatile int mc_expected, mc_received;
static void ia64_dispatch_intr(void *, u_int);
static void
dummy_perf(unsigned long vector, struct trapframe *tf)
@ -127,7 +128,7 @@ SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD,
&adjust_ticks, 0, "Total number of ITC interrupts with adjustment");
int
interrupt(u_int64_t vector, struct trapframe *tf)
interrupt(u_int vector, struct trapframe *tf)
{
struct thread *td;
volatile struct ia64_interrupt_block *ib = IA64_INTERRUPT_BLOCK;
@ -146,7 +147,7 @@ interrupt(u_int64_t vector, struct trapframe *tf)
*/
if (vector == 0) {
vector = ib->ib_inta;
printf("ExtINT interrupt: vector=%ld\n", vector);
printf("ExtINT interrupt: vector=%u\n", vector);
if (vector == 15)
goto stray;
}
@ -251,93 +252,115 @@ interrupt(u_int64_t vector, struct trapframe *tf)
#define IA64_HARDWARE_IRQ_BASE 0x20
struct ia64_intr {
struct intr_event *event; /* interrupt event */
volatile long *cntp; /* interrupt counter */
struct intr_event *event; /* interrupt event */
volatile long *cntp; /* interrupt counter */
struct sapic *sapic;
u_int irq;
};
static struct mtx ia64_intrs_lock;
static struct ia64_intr *ia64_intrs[256];
extern struct sapic *ia64_sapics[];
extern int ia64_sapic_count;
static void
ithds_init(void *dummy)
ia64_intr_eoi(void *arg)
{
u_int vector = (uintptr_t)arg;
struct ia64_intr *i;
mtx_init(&ia64_intrs_lock, "intr table", NULL, MTX_SPIN);
i = ia64_intrs[vector];
if (i != NULL)
sapic_eoi(i->sapic, vector);
}
SYSINIT(ithds_init, SI_SUB_INTR, SI_ORDER_SECOND, ithds_init, NULL);
#ifdef INTR_FILTER
static void
ia64_send_eoi(uintptr_t vector)
ia64_intr_disable(void *arg)
{
int irq, i;
u_int vector = (uintptr_t)arg;
struct ia64_intr *i;
irq = vector - IA64_HARDWARE_IRQ_BASE;
for (i = 0; i < ia64_sapic_count; i++) {
struct sapic *sa = ia64_sapics[i];
if (irq >= sa->sa_base && irq <= sa->sa_limit)
sapic_eoi(sa, vector);
i = ia64_intrs[vector];
if (i != NULL) {
sapic_eoi(i->sapic, vector);
sapic_mask(i->sapic, i->irq);
}
}
static void
ia64_intr_enable(void *arg)
{
u_int vector = (uintptr_t)arg;
struct ia64_intr *i;
i = ia64_intrs[vector];
if (i != NULL)
sapic_unmask(i->sapic, i->irq);
}
#endif
int
ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
void **cookiep, volatile long *cntp)
ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
{
struct ia64_intr *i;
int errcode;
intptr_t vector = irq + IA64_HARDWARE_IRQ_BASE;
struct sapic *sa;
char *intrname;
u_int vector;
int error;
/* Get the I/O SAPIC that corresponds to the IRQ. */
sa = sapic_lookup(irq);
if (sa == NULL)
return (EINVAL);
/*
* XXX - Can we have more than one device on a vector? If so, we have
* a race condition here that needs to be worked around similar to
* the fashion done in the i386 inthand_add() function.
* XXX - There's a priority implied by the choice of vector.
* We should therefore relate the vector to the interrupt type.
*/
/* First, check for an existing hash table entry for this vector. */
mtx_lock_spin(&ia64_intrs_lock);
i = ia64_intrs[vector];
mtx_unlock_spin(&ia64_intrs_lock);
vector = irq + IA64_HARDWARE_IRQ_BASE;
i = ia64_intrs[vector];
if (i == NULL) {
/* None was found, so create an entry. */
i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_NOWAIT);
if (i == NULL)
return ENOMEM;
if (cntp == NULL)
i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
else
i->cntp = cntp;
if (name != NULL && *name != '\0') {
/* XXX needs abstraction. Too error phrone. */
intrname = intrnames + (irq + INTRCNT_ISA_IRQ) *
INTRNAME_LEN;
memset(intrname, ' ', INTRNAME_LEN - 1);
bcopy(name, intrname, strlen(name));
}
errcode = intr_event_create(&i->event, (void *)vector, 0,
(void (*)(void *))ia64_send_eoi, "intr:");
if (errcode) {
return (ENOMEM);
#ifdef INTR_FILTER
error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
0, ia64_intr_enable, ia64_intr_eoi, ia64_intr_disable,
"irq%u:", irq);
#else
error = intr_event_create(&i->event, (void *)(uintptr_t)vector,
0, ia64_intr_eoi, "irq%u:", irq);
#endif
if (error) {
free(i, M_DEVBUF);
return errcode;
return (error);
}
mtx_lock_spin(&ia64_intrs_lock);
ia64_intrs[vector] = i;
mtx_unlock_spin(&ia64_intrs_lock);
if (!atomic_cmpset_ptr(&ia64_intrs[vector], NULL, i)) {
intr_event_destroy(i->event);
free(i, M_DEVBUF);
i = ia64_intrs[vector];
} else {
i->sapic = sa;
i->irq = irq;
i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ;
if (name != NULL && *name != '\0') {
/* XXX needs abstraction. Too error prone. */
intrname = intrnames +
(irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN;
memset(intrname, ' ', INTRNAME_LEN - 1);
bcopy(name, intrname, strlen(name));
}
sapic_enable(i->sapic, irq, vector);
}
}
/* Second, add this handler. */
errcode = intr_event_add_handler(i->event, name, filter, handler, arg,
error = intr_event_add_handler(i->event, name, filter, handler, arg,
intr_priority(flags), flags, cookiep);
if (errcode)
return errcode;
return (sapic_enable(irq, vector));
return (error);
}
int
@ -347,27 +370,33 @@ ia64_teardown_intr(void *cookie)
return (intr_event_remove_handler(cookie));
}
void
ia64_dispatch_intr(void *frame, unsigned long vector)
static void
ia64_dispatch_intr(void *frame, u_int vector)
{
struct ia64_intr *i;
struct intr_event *ie; /* our interrupt event */
#ifndef INTR_FILTER
struct intr_handler *ih;
int error, thread, ret;
#endif
/*
* Find the interrupt thread for this vector.
*/
i = ia64_intrs[vector];
if (i == NULL)
return; /* no event for this vector */
KASSERT(i != NULL, ("%s: unassigned vector", __func__));
if (i->cntp)
atomic_add_long(i->cntp, 1);
(*i->cntp)++;
ie = i->event;
KASSERT(ie != NULL, ("interrupt vector without an event"));
KASSERT(ie != NULL, ("%s: interrupt without event", __func__));
#ifdef INTR_FILTER
if (intr_event_handle(ie, frame) != 0) {
ia64_intr_disable((void *)(uintptr_t)vector);
log(LOG_ERR, "stray irq%u\n", i->irq);
}
#else
/*
* As an optimization, if an event has no handlers, don't
* schedule it to run.
@ -403,37 +432,41 @@ ia64_dispatch_intr(void *frame, unsigned long vector)
if (thread) {
error = intr_event_schedule_thread(ie);
KASSERT(error == 0, ("got an impossible stray interrupt"));
KASSERT(error == 0, ("%s: impossible stray", __func__));
} else
ia64_send_eoi(vector);
ia64_intr_eoi((void *)(uintptr_t)vector);
#endif
}
#ifdef DDB
static void
db_show_vector(int vector)
db_print_vector(u_int vector, int always)
{
int irq, i;
struct ia64_intr *i;
irq = vector - IA64_HARDWARE_IRQ_BASE;
for (i = 0; i < ia64_sapic_count; i++) {
struct sapic *sa = ia64_sapics[i];
if (irq >= sa->sa_base && irq <= sa->sa_limit)
sapic_print(sa, irq - sa->sa_base);
}
i = ia64_intrs[vector];
if (i != NULL) {
db_printf("vector %u (%p): ", vector, i);
sapic_print(i->sapic, i->irq);
} else if (always)
db_printf("vector %u: unassigned\n", vector);
}
DB_SHOW_COMMAND(irq, db_show_irq)
DB_SHOW_COMMAND(vector, db_show_vector)
{
int vector;
u_int vector;
if (have_addr) {
vector = ((addr >> 4) % 16) * 10 + (addr % 16);
db_show_vector(vector);
if (vector >= 256)
db_printf("error: vector %u not in range [0..255]\n",
vector);
else
db_print_vector(vector, 1);
} else {
for (vector = IA64_HARDWARE_IRQ_BASE;
vector < IA64_HARDWARE_IRQ_BASE + 64; vector++)
db_show_vector(vector);
for (vector = 0; vector < 256; vector++)
db_print_vector(vector, 0);
}
}

View File

@ -447,7 +447,7 @@ nexus_setup_intr(device_t bus, device_t child, struct resource *irq,
return (error);
error = ia64_setup_intr(device_get_nameunit(child),
rman_get_start(irq), filter, ihand, arg, flags, cookiep, 0);
rman_get_start(irq), filter, ihand, arg, flags, cookiep);
return (error);
}

View File

@ -33,6 +33,8 @@
#include <sys/kernel.h>
#include <sys/systm.h>
#include <sys/bus.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>
#include <machine/intr.h>
@ -67,7 +69,22 @@ struct sapic_rte {
u_int64_t rte_destination_id :8;
};
static u_int32_t
struct sapic *
sapic_lookup(u_int irq)
{
struct sapic *sa;
int i;
for (i = 0; i < ia64_sapic_count; i++) {
sa = ia64_sapics[i];
if (irq >= sa->sa_base && irq <= sa->sa_limit)
return (sa);
}
return (NULL);
}
static __inline u_int32_t
sapic_read(struct sapic *sa, int which)
{
vm_offset_t reg = sa->sa_registers;
@ -77,7 +94,7 @@ sapic_read(struct sapic *sa, int which)
return *(volatile u_int32_t *) (reg + SAPIC_IO_WINDOW);
}
static void
static __inline void
sapic_write(struct sapic *sa, int which, u_int32_t value)
{
vm_offset_t reg = sa->sa_registers;
@ -88,77 +105,70 @@ sapic_write(struct sapic *sa, int which, u_int32_t value)
ia64_mf();
}
static void
static __inline void
sapic_read_rte(struct sapic *sa, int which, struct sapic_rte *rte)
{
u_int32_t *p = (u_int32_t *) rte;
register_t c;
c = intr_disable();
p[0] = sapic_read(sa, SAPIC_RTE_BASE + 2*which);
p[1] = sapic_read(sa, SAPIC_RTE_BASE + 2*which + 1);
intr_restore(c);
p[0] = sapic_read(sa, SAPIC_RTE_BASE + 2 * which);
p[1] = sapic_read(sa, SAPIC_RTE_BASE + 2 * which + 1);
}
static void
static __inline void
sapic_write_rte(struct sapic *sa, int which, struct sapic_rte *rte)
{
u_int32_t *p = (u_int32_t *) rte;
register_t c;
c = intr_disable();
sapic_write(sa, SAPIC_RTE_BASE + 2*which, p[0]);
sapic_write(sa, SAPIC_RTE_BASE + 2*which + 1, p[1]);
intr_restore(c);
sapic_write(sa, SAPIC_RTE_BASE + 2 *which, p[0]);
sapic_write(sa, SAPIC_RTE_BASE + 2 *which + 1, p[1]);
}
int
sapic_config_intr(int irq, enum intr_trigger trig, enum intr_polarity pol)
sapic_config_intr(u_int irq, enum intr_trigger trig, enum intr_polarity pol)
{
struct sapic_rte rte;
struct sapic *sa;
int i;
for (i = 0; i < ia64_sapic_count; i++) {
sa = ia64_sapics[i];
if (irq < sa->sa_base || irq > sa->sa_limit)
continue;
sa = sapic_lookup(irq);
if (sa == NULL)
return (EINVAL);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
if (trig != INTR_TRIGGER_CONFORM)
rte.rte_trigger_mode = (trig == INTR_TRIGGER_EDGE) ?
SAPIC_TRIGGER_EDGE : SAPIC_TRIGGER_LEVEL;
else
rte.rte_trigger_mode = (irq < 16) ?
SAPIC_TRIGGER_EDGE : SAPIC_TRIGGER_LEVEL;
if (pol != INTR_POLARITY_CONFORM)
rte.rte_polarity = (pol == INTR_POLARITY_HIGH) ?
SAPIC_POLARITY_HIGH : SAPIC_POLARITY_LOW;
else
rte.rte_polarity = (irq < 16) ? SAPIC_POLARITY_HIGH :
SAPIC_POLARITY_LOW;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
return (0);
}
return (ENOENT);
mtx_lock_spin(&sa->sa_mtx);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
if (trig != INTR_TRIGGER_CONFORM)
rte.rte_trigger_mode = (trig == INTR_TRIGGER_EDGE) ?
SAPIC_TRIGGER_EDGE : SAPIC_TRIGGER_LEVEL;
else
rte.rte_trigger_mode = (irq < 16) ? SAPIC_TRIGGER_EDGE :
SAPIC_TRIGGER_LEVEL;
if (pol != INTR_POLARITY_CONFORM)
rte.rte_polarity = (pol == INTR_POLARITY_HIGH) ?
SAPIC_POLARITY_HIGH : SAPIC_POLARITY_LOW;
else
rte.rte_polarity = (irq < 16) ? SAPIC_POLARITY_HIGH :
SAPIC_POLARITY_LOW;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
mtx_unlock_spin(&sa->sa_mtx);
return (0);
}
struct sapic *
sapic_create(int id, int base, u_int64_t address)
sapic_create(u_int id, u_int base, u_int64_t address)
{
struct sapic_rte rte;
struct sapic *sa;
int i, max;
u_int i, max;
sa = malloc(sizeof(struct sapic), M_SAPIC, M_NOWAIT);
if (!sa)
return 0;
sa = malloc(sizeof(struct sapic), M_SAPIC, M_ZERO | M_NOWAIT);
if (sa == NULL)
return (NULL);
sa->sa_id = id;
sa->sa_base = base;
sa->sa_registers = IA64_PHYS_TO_RR6(address);
mtx_init(&sa->sa_mtx, "I/O SAPIC lock", NULL, MTX_SPIN);
max = (sapic_read(sa, SAPIC_VERSION) >> 16) & 0xff;
sa->sa_limit = base + max;
@ -183,39 +193,58 @@ sapic_create(int id, int base, u_int64_t address)
}
int
sapic_enable(int irq, int vector)
sapic_enable(struct sapic *sa, u_int irq, u_int vector)
{
struct sapic_rte rte;
struct sapic *sa;
uint64_t lid = ia64_get_lid();
int i;
for (i = 0; i < ia64_sapic_count; i++) {
sa = ia64_sapics[i];
if (irq < sa->sa_base || irq > sa->sa_limit)
continue;
sapic_read_rte(sa, irq - sa->sa_base, &rte);
rte.rte_destination_id = (lid >> 24) & 255;
rte.rte_destination_eid = (lid >> 16) & 255;
rte.rte_delivery_mode = SAPIC_DELMODE_LOWPRI;
rte.rte_vector = vector;
rte.rte_mask = 0;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
return (0);
}
return (ENOENT);
mtx_lock_spin(&sa->sa_mtx);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
rte.rte_destination_id = (lid >> 24) & 255;
rte.rte_destination_eid = (lid >> 16) & 255;
rte.rte_delivery_mode = SAPIC_DELMODE_LOWPRI;
rte.rte_vector = vector;
rte.rte_mask = 0;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
mtx_unlock_spin(&sa->sa_mtx);
return (0);
}
void
sapic_eoi(struct sapic *sa, int vector)
sapic_eoi(struct sapic *sa, u_int vector)
{
vm_offset_t reg = sa->sa_registers;
*(volatile u_int32_t *) (reg + SAPIC_APIC_EOI) = vector;
*(volatile u_int32_t *)(reg + SAPIC_APIC_EOI) = vector;
ia64_mf();
}
/* Expected to be called with interrupts disabled. */
void
sapic_mask(struct sapic *sa, u_int irq)
{
struct sapic_rte rte;
mtx_lock_spin(&sa->sa_mtx);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
rte.rte_mask = 1;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
mtx_unlock_spin(&sa->sa_mtx);
}
/* Expected to be called with interrupts disabled. */
void
sapic_unmask(struct sapic *sa, u_int irq)
{
struct sapic_rte rte;
mtx_lock_spin(&sa->sa_mtx);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
rte.rte_mask = 0;
sapic_write_rte(sa, irq - sa->sa_base, &rte);
mtx_unlock_spin(&sa->sa_mtx);
}
static int
sysctl_machdep_apic(SYSCTL_HANDLER_ARGS)
{
@ -233,8 +262,10 @@ sysctl_machdep_apic(SYSCTL_HANDLER_ARGS)
sa = ia64_sapics[apic];
count = sa->sa_limit - sa->sa_base + 1;
for (index = 0; index < count; index++) {
mtx_lock_spin(&sa->sa_mtx);
sapic_read_rte(sa, index, &rte);
if (rte.rte_mask != 0)
mtx_unlock_spin(&sa->sa_mtx);
if (rte.rte_vector == 0)
continue;
len = sprintf(buf,
" 0x%02x %3d: (%02x,%02x): %3d %d %d %s %s %s %s %s\n",
@ -261,24 +292,22 @@ sysctl_machdep_apic(SYSCTL_HANDLER_ARGS)
#include <ddb/ddb.h>
void
sapic_print(struct sapic *sa, int input)
sapic_print(struct sapic *sa, u_int irq)
{
struct sapic_rte rte;
sapic_read_rte(sa, input, &rte);
if (rte.rte_mask == 0) {
db_printf("%3d %d %d %s %s %s %s %s ID=%x EID=%x\n",
rte.rte_vector,
rte.rte_delivery_mode,
rte.rte_destination_mode,
rte.rte_delivery_status ? "DS" : " ",
rte.rte_polarity ? "low-active " : "high-active",
rte.rte_rirr ? "RIRR" : " ",
rte.rte_trigger_mode ? "level" : "edge ",
rte.rte_flushen ? "F" : " ",
rte.rte_destination_id,
rte.rte_destination_eid);
}
db_printf("sapic=%u, irq=%u: ", sa->sa_id, irq);
sapic_read_rte(sa, irq - sa->sa_base, &rte);
db_printf("%3d %x->%x:%x %d %s %s %s %s %s %s\n", rte.rte_vector,
rte.rte_delivery_mode,
rte.rte_destination_id, rte.rte_destination_eid,
rte.rte_destination_mode,
rte.rte_delivery_status ? "DS" : " ",
rte.rte_polarity ? "low-active " : "high-active",
rte.rte_rirr ? "RIRR" : " ",
rte.rte_trigger_mode ? "level" : "edge ",
rte.rte_flushen ? "F" : " ",
rte.rte_mask ? "(masked)" : "");
}
#endif

View File

@ -48,13 +48,8 @@ extern u_int64_t ia64_lapic_address;
#define IA64_INTERRUPT_BLOCK \
(struct ia64_interrupt_block *)IA64_PHYS_TO_RR6(ia64_lapic_address)
struct sapic;
void ia64_add_sapic(struct sapic *sa);
int ia64_setup_intr(const char *name, int irq, driver_filter_t filter,
driver_intr_t handler, void *arg, enum intr_type flags,
void **cookiep, volatile long *cntp);
driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep);
int ia64_teardown_intr(void *cookie);
void ia64_dispatch_intr(void *frame, unsigned long vector);
#endif /* !_MACHINE_INTR_H_ */

View File

@ -84,7 +84,7 @@ int ia64_highfp_drop(struct thread *);
int ia64_highfp_save(struct thread *);
void ia64_init(void);
void ia64_probe_sapics(void);
int interrupt(uint64_t, struct trapframe *);
int interrupt(u_int, struct trapframe *);
void map_gateway_page(void);
void map_pal_code(void);
void map_vhpt(uintptr_t);

View File

@ -30,10 +30,11 @@
#define _MACHINE_SAPICVAR_H_
struct sapic {
int sa_id; /* I/O SAPIC Id */
int sa_base; /* ACPI vector base */
int sa_limit; /* last ACPI vector handled here */
struct mtx sa_mtx;
vm_offset_t sa_registers; /* virtual address of sapic */
u_int sa_id; /* I/O SAPIC Id */
u_int sa_base; /* ACPI vector base */
u_int sa_limit; /* last ACPI vector handled here */
};
#define SAPIC_TRIGGER_EDGE 0
@ -49,12 +50,16 @@ struct sapic {
#define SAPIC_DELMODE_INIT 5
#define SAPIC_DELMODE_EXTINT 7
int sapic_config_intr(int irq, enum intr_trigger, enum intr_polarity);
struct sapic *sapic_create(int id, int base, uint64_t address);
int sapic_enable(int irq, int vector);
void sapic_eoi(struct sapic *sa, int vector);
int sapic_config_intr(u_int irq, enum intr_trigger, enum intr_polarity);
struct sapic *sapic_create(u_int id, u_int base, uint64_t address);
int sapic_enable(struct sapic *sa, u_int irq, u_int vector);
void sapic_eoi(struct sapic *sa, u_int vector);
struct sapic *sapic_lookup(u_int irq);
void sapic_mask(struct sapic *sa, u_int irq);
void sapic_unmask(struct sapic *sa, u_int irq);
#ifdef DDB
void sapic_print(struct sapic *sa, int input);
void sapic_print(struct sapic *sa, u_int irq);
#endif
#endif /* ! _MACHINE_SAPICVAR_H_ */