diff --git a/sys/ia64/ia64/clock.c b/sys/ia64/ia64/clock.c index a9c39fce3598..0afbc8788f85 100644 --- a/sys/ia64/ia64/clock.c +++ b/sys/ia64/ia64/clock.c @@ -29,19 +29,41 @@ __FBSDID("$FreeBSD$"); #include #include +#include +#include +#include #include #include #include -#include #include #include -#include #include #include +#include +#include #include -uint64_t ia64_clock_reload; +SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics"); + +static int adjust_edges = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD, + &adjust_edges, 0, "Number of times ITC got more than 12.5% behind"); + +static int adjust_excess = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD, + &adjust_excess, 0, "Total number of ignored ITC interrupts"); + +static int adjust_lost = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD, + &adjust_lost, 0, "Total number of lost ITC interrupts"); + +static int adjust_ticks = 0; +SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD, + &adjust_ticks, 0, "Total number of ITC interrupts with adjustment"); + +static u_int ia64_clock_xiv; +static uint64_t ia64_clock_reload; #ifndef SMP static timecounter_get_t ia64_get_timecount; @@ -54,13 +76,68 @@ static struct timecounter ia64_timecounter = { "ITC" /* name */ }; -static unsigned +static u_int ia64_get_timecount(struct timecounter* tc) { return ia64_get_itc(); } #endif +static u_int +ia64_ih_clock(struct thread *td, u_int xiv, struct trapframe *tf) +{ + uint64_t adj, clk, itc; + int64_t delta; + int count; + + ia64_set_eoi(0); + + PCPU_INC(md.stats.pcs_nclks); + intrcnt[INTRCNT_CLOCK]++; + + ia64_srlz_d(); + + itc = ia64_get_itc(); + + adj = PCPU_GET(md.clockadj); + clk = PCPU_GET(md.clock); + + delta = itc - clk; + count = 0; + while (delta >= ia64_clock_reload) { + /* Only the BSP runs the real clock */ + if (PCPU_GET(cpuid) == 0) + hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); + else + hardclock_cpu(TRAPF_USERMODE(tf)); + if (profprocs != 0) + profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); + statclock(TRAPF_USERMODE(tf)); + delta -= ia64_clock_reload; + clk += ia64_clock_reload; + if (adj != 0) + adjust_ticks++; + count++; + } + ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj); + if (count > 0) { + adjust_lost += count - 1; + if (delta > (ia64_clock_reload >> 3)) { + if (adj == 0) + adjust_edges++; + adj = ia64_clock_reload >> 4; + } else + adj = 0; + } else { + adj = 0; + adjust_excess++; + } + PCPU_SET(md.clock, clk); + PCPU_SET(md.clockadj, adj); + ia64_srlz_d(); + return (0); +} + void pcpu_initclock(void) { @@ -68,7 +145,7 @@ pcpu_initclock(void) PCPU_SET(md.clockadj, 0); PCPU_SET(md.clock, ia64_get_itc()); ia64_set_itm(PCPU_GET(md.clock) + ia64_clock_reload); - ia64_set_itv(CLOCK_VECTOR); /* highest priority class */ + ia64_set_itv(ia64_clock_xiv); ia64_srlz_d(); } @@ -81,6 +158,11 @@ cpu_initclocks() { u_long itc_freq; + ia64_clock_xiv = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IRQ, + ia64_ih_clock); + if (ia64_clock_xiv == 0) + panic("No XIV for clock interrupts"); + itc_freq = (u_long)ia64_itc_freq() * 1000000ul; stathz = hz; diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S index 2868c8a3f771..5e186f9be929 100644 --- a/sys/ia64/ia64/exception.S +++ b/sys/ia64/ia64/exception.S @@ -1303,7 +1303,7 @@ IVT_END(Break_Instruction) IVT_ENTRY(External_Interrupt, 0x3000) { .mib - mov r17=cr.ivr // Put the vector in the trap frame. + mov r17=cr.ivr // Put the XIV in the trapframe. mov r16=ip br.sptk exception_save ;; @@ -1317,7 +1317,7 @@ IVT_ENTRY(External_Interrupt, 0x3000) { .mib add out0=16,sp nop 0 - br.call.sptk rp=interrupt + br.call.sptk rp=ia64_handle_intr ;; } { .mib diff --git a/sys/ia64/ia64/highfp.c b/sys/ia64/ia64/highfp.c index 145ee488ba66..3aa3af0e4a0a 100644 --- a/sys/ia64/ia64/highfp.c +++ b/sys/ia64/ia64/highfp.c @@ -53,7 +53,7 @@ ia64_highfp_ipi(struct pcpu *cpu) { int error; - ipi_send(cpu, IPI_HIGH_FP); + ipi_send(cpu, ia64_ipi_highfp); error = msleep_spin(&cpu->pc_fpcurthread, &ia64_highfp_mtx, "High FP", 0); return (error); diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c index a9bee2798df0..ad8a4e3ceda5 100644 --- a/sys/ia64/ia64/interrupt.c +++ b/sys/ia64/ia64/interrupt.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -52,46 +53,20 @@ #include #include -#include #include #include #include #include +#include #include #include #include #include -#ifdef EVCNT_COUNTERS -struct evcnt clock_intr_evcnt; /* event counter for clock intrs. */ -#else -#include -#include -#endif - #ifdef DDB #include #endif -SYSCTL_NODE(_debug, OID_AUTO, clock, CTLFLAG_RW, 0, "clock statistics"); - -static int adjust_edges = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_edges, CTLFLAG_RD, - &adjust_edges, 0, "Number of times ITC got more than 12.5% behind"); - -static int adjust_excess = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_excess, CTLFLAG_RD, - &adjust_excess, 0, "Total number of ignored ITC interrupts"); - -static int adjust_lost = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_lost, CTLFLAG_RD, - &adjust_lost, 0, "Total number of lost ITC interrupts"); - -static int adjust_ticks = 0; -SYSCTL_INT(_debug_clock, OID_AUTO, adjust_ticks, CTLFLAG_RD, - &adjust_ticks, 0, "Total number of ITC interrupts with adjustment"); - - struct ia64_intr { struct intr_event *event; /* interrupt event */ volatile long *cntp; /* interrupt counter */ @@ -99,190 +74,120 @@ struct ia64_intr { u_int irq; }; -static struct ia64_intr *ia64_intrs[256]; +ia64_ihtype *ia64_handler[IA64_NXIVS]; +static enum ia64_xiv_use ia64_xiv[IA64_NXIVS]; +static struct ia64_intr *ia64_intrs[IA64_NXIVS]; -static void ia64_dispatch_intr(void *, u_int); +static ia64_ihtype ia64_ih_invalid; +static ia64_ihtype ia64_ih_irq; void -interrupt(struct trapframe *tf) +ia64_xiv_init(void) { - struct thread *td; - uint64_t adj, clk, itc; - int64_t delta; - u_int vector; - int count; - uint8_t inta; + u_int xiv; - ia64_set_fpsr(IA64_FPSR_DEFAULT); - - td = curthread; - - PCPU_INC(cnt.v_intr); - - vector = tf->tf_special.ifa; - - next: - /* - * Handle ExtINT interrupts by generating an INTA cycle to - * read the vector. - */ - if (vector == 0) { - PCPU_INC(md.stats.pcs_nextints); - inta = ia64_ld1(&ia64_pib->ib_inta); - if (inta == 15) { - PCPU_INC(md.stats.pcs_nstrays); - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - goto stray; - } - vector = (int)inta; - } else if (vector == 15) { - PCPU_INC(md.stats.pcs_nstrays); - goto stray; - } - - if (vector == CLOCK_VECTOR) {/* clock interrupt */ - /* CTR0(KTR_INTR, "clock interrupt"); */ - - itc = ia64_get_itc(); - - PCPU_INC(md.stats.pcs_nclks); -#ifdef EVCNT_COUNTERS - clock_intr_evcnt.ev_count++; -#else - intrcnt[INTRCNT_CLOCK]++; -#endif - - critical_enter(); - - adj = PCPU_GET(md.clockadj); - clk = PCPU_GET(md.clock); - delta = itc - clk; - count = 0; - while (delta >= ia64_clock_reload) { - /* Only the BSP runs the real clock */ - if (PCPU_GET(cpuid) == 0) - hardclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); - else - hardclock_cpu(TRAPF_USERMODE(tf)); - if (profprocs != 0) - profclock(TRAPF_USERMODE(tf), TRAPF_PC(tf)); - statclock(TRAPF_USERMODE(tf)); - delta -= ia64_clock_reload; - clk += ia64_clock_reload; - if (adj != 0) - adjust_ticks++; - count++; - } - ia64_set_itm(ia64_get_itc() + ia64_clock_reload - adj); - if (count > 0) { - adjust_lost += count - 1; - if (delta > (ia64_clock_reload >> 3)) { - if (adj == 0) - adjust_edges++; - adj = ia64_clock_reload >> 4; - } else - adj = 0; - } else { - adj = 0; - adjust_excess++; - } - PCPU_SET(md.clock, clk); - PCPU_SET(md.clockadj, adj); - critical_exit(); - ia64_srlz_d(); - -#ifdef SMP - } else if (vector == ipi_vector[IPI_AST]) { - PCPU_INC(md.stats.pcs_nasts); - CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); - } else if (vector == ipi_vector[IPI_HIGH_FP]) { - PCPU_INC(md.stats.pcs_nhighfps); - ia64_highfp_save_ipi(); - } else if (vector == ipi_vector[IPI_RENDEZVOUS]) { - PCPU_INC(md.stats.pcs_nrdvs); - CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); - enable_intr(); - smp_rendezvous_action(); - disable_intr(); - } else if (vector == ipi_vector[IPI_STOP]) { - PCPU_INC(md.stats.pcs_nstops); - cpumask_t mybit = PCPU_GET(cpumask); - - /* Make sure IPI_STOP_HARD is mapped to IPI_STOP. */ - KASSERT(IPI_STOP == IPI_STOP_HARD, - ("%s: IPI_STOP_HARD not handled.", __func__)); - - savectx(PCPU_PTR(md.pcb)); - atomic_set_int(&stopped_cpus, mybit); - while ((started_cpus & mybit) == 0) - cpu_spinwait(); - atomic_clear_int(&started_cpus, mybit); - atomic_clear_int(&stopped_cpus, mybit); - } else if (vector == ipi_vector[IPI_PREEMPT]) { - PCPU_INC(md.stats.pcs_npreempts); - CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid)); - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - enable_intr(); - sched_preempt(curthread); - disable_intr(); - goto stray; -#endif - } else { - PCPU_INC(md.stats.pcs_nhwints); - atomic_add_int(&td->td_intr_nesting_level, 1); - ia64_dispatch_intr(tf, vector); - atomic_subtract_int(&td->td_intr_nesting_level, 1); - } - - __asm __volatile("mov cr.eoi = r0;; srlz.d"); - vector = ia64_get_ivr(); - if (vector != 15) - goto next; - -stray: - if (TRAPF_USERMODE(tf)) { - enable_intr(); - userret(td, tf); - mtx_assert(&Giant, MA_NOTOWNED); - do_ast(tf); + for (xiv = 0; xiv < IA64_NXIVS; xiv++) { + ia64_handler[xiv] = ia64_ih_invalid; + ia64_xiv[xiv] = IA64_XIV_FREE; + ia64_intrs[xiv] = NULL; } + (void)ia64_xiv_reserve(15, IA64_XIV_ARCH, NULL); } +int +ia64_xiv_free(u_int xiv, enum ia64_xiv_use what) +{ + + if (xiv >= IA64_NXIVS) + return (EINVAL); + if (what == IA64_XIV_FREE || what == IA64_XIV_ARCH) + return (EINVAL); + if (ia64_xiv[xiv] != what) + return (ENXIO); + ia64_xiv[xiv] = IA64_XIV_FREE; + ia64_handler[xiv] = ia64_ih_invalid; + return (0); +} + +int +ia64_xiv_reserve(u_int xiv, enum ia64_xiv_use what, ia64_ihtype ih) +{ + + if (xiv >= IA64_NXIVS) + return (EINVAL); + if (what == IA64_XIV_FREE) + return (EINVAL); + if (ia64_xiv[xiv] != IA64_XIV_FREE) + return (EBUSY); + ia64_xiv[xiv] = what; + ia64_handler[xiv] = (ih == NULL) ? ia64_ih_invalid: ih; + if (1 || bootverbose) + printf("XIV %u: use=%u, IH=%p\n", xiv, what, ih); + return (0); +} + +u_int +ia64_xiv_alloc(u_int prio, enum ia64_xiv_use what, ia64_ihtype ih) +{ + u_int hwprio; + u_int xiv0, xiv; + + hwprio = prio >> 2; + if (hwprio > IA64_MAX_HWPRIO) + hwprio = IA64_MAX_HWPRIO; + + xiv0 = IA64_NXIVS - (hwprio + 1) * 16; + + KASSERT(xiv0 > IA64_MIN_XIV, ("%s: min XIV", __func__)); + KASSERT(xiv0 < IA64_NXIVS, ("%s: max XIV", __func__)); + + xiv = xiv0; + while (xiv < IA64_NXIVS && ia64_xiv_reserve(xiv, what, ih)) + xiv++; + + if (xiv < IA64_NXIVS) + return (xiv); + + xiv = xiv0; + while (xiv >= IA64_MIN_XIV && ia64_xiv_reserve(xiv, what, ih)) + xiv--; + + return ((xiv >= IA64_MIN_XIV) ? xiv : 0); +} static void ia64_intr_eoi(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) - sapic_eoi(i->sapic, vector); + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_eoi(i->sapic, xiv); } static void ia64_intr_mask(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) { - sapic_mask(i->sapic, i->irq); - sapic_eoi(i->sapic, vector); - } + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_mask(i->sapic, i->irq); + sapic_eoi(i->sapic, xiv); } static void ia64_intr_unmask(void *arg) { - u_int vector = (uintptr_t)arg; + u_int xiv = (uintptr_t)arg; struct ia64_intr *i; - i = ia64_intrs[vector]; - if (i != NULL) - sapic_unmask(i->sapic, i->irq); + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s", __func__)); + sapic_unmask(i->sapic, i->irq); } int @@ -292,7 +197,7 @@ ia64_setup_intr(const char *name, int irq, driver_filter_t filter, struct ia64_intr *i; struct sapic *sa; char *intrname; - u_int prio, vector; + u_int prio, xiv; int error; prio = intr_priority(flags); @@ -301,37 +206,41 @@ ia64_setup_intr(const char *name, int irq, driver_filter_t filter, /* XXX lock */ - /* Get the I/O SAPIC and vector that corresponds to the IRQ. */ - sa = sapic_lookup(irq, &vector); + /* Get the I/O SAPIC and XIV that corresponds to the IRQ. */ + sa = sapic_lookup(irq, &xiv); if (sa == NULL) { /* XXX unlock */ return (EINVAL); } - if (vector == 0) { + if (xiv == 0) { /* XXX unlock */ i = malloc(sizeof(struct ia64_intr), M_DEVBUF, M_ZERO | M_WAITOK); /* XXX lock */ - sa = sapic_lookup(irq, &vector); + sa = sapic_lookup(irq, &xiv); KASSERT(sa != NULL, ("sapic_lookup")); - if (vector != 0) + if (xiv != 0) free(i, M_DEVBUF); } /* - * If the IRQ has no vector assigned to it yet, assign one based + * If the IRQ has no XIV assigned to it yet, assign one based * on the priority. */ - if (vector == 0) { - vector = (256 - 64) - (prio << 1); - while (vector < 256 && ia64_intrs[vector] != NULL) - vector++; + if (xiv == 0) { + xiv = ia64_xiv_alloc(prio, IA64_XIV_IRQ, ia64_ih_irq); + if (xiv == 0) { + /* XXX unlock */ + free(i, M_DEVBUF); + return (ENOSPC); + } - error = intr_event_create(&i->event, (void *)(uintptr_t)vector, + error = intr_event_create(&i->event, (void *)(uintptr_t)xiv, 0, irq, ia64_intr_mask, ia64_intr_unmask, ia64_intr_eoi, NULL, "irq%u:", irq); if (error) { + ia64_xiv_free(xiv, IA64_XIV_IRQ); /* XXX unlock */ free(i, M_DEVBUF); return (error); @@ -339,25 +248,25 @@ ia64_setup_intr(const char *name, int irq, driver_filter_t filter, i->sapic = sa; i->irq = irq; - i->cntp = intrcnt + irq + INTRCNT_ISA_IRQ; - ia64_intrs[vector] = i; - sapic_enable(sa, irq, vector); + i->cntp = intrcnt + xiv; + ia64_intrs[xiv] = i; /* XXX unlock */ + sapic_enable(sa, irq, xiv); + if (name != NULL && *name != '\0') { /* XXX needs abstraction. Too error prone. */ - intrname = intrnames + - (irq + INTRCNT_ISA_IRQ) * INTRNAME_LEN; + intrname = intrnames + xiv * INTRNAME_LEN; memset(intrname, ' ', INTRNAME_LEN - 1); bcopy(name, intrname, strlen(name)); } } else { - i = ia64_intrs[vector]; + i = ia64_intrs[xiv]; /* XXX unlock */ } - KASSERT(i != NULL, ("vector mapping bug")); + KASSERT(i != NULL, ("XIV mapping bug")); error = intr_event_add_handler(i->event, name, filter, handler, arg, prio, flags, cookiep); @@ -371,62 +280,114 @@ ia64_teardown_intr(void *cookie) return (intr_event_remove_handler(cookie)); } -static void -ia64_dispatch_intr(void *frame, u_int vector) +/* + * Interrupt handlers. + */ + +void +ia64_handle_intr(struct trapframe *tf) +{ + struct thread *td; + u_int rfi, xiv; + + td = curthread; + ia64_set_fpsr(IA64_FPSR_DEFAULT); + PCPU_INC(cnt.v_intr); + + xiv = tf->tf_special.ifa; + if (xiv == 15) { + PCPU_INC(md.stats.pcs_nstrays); + goto out; + } + + while (xiv != 15) { + CTR1(KTR_INTR, "INTR: XIV=%u", xiv); + critical_enter(); + rfi = (ia64_handler[xiv])(td, xiv, tf); + if (rfi) { + critical_exit(); + return; + } + xiv = ia64_get_ivr(); + critical_exit(); + ia64_srlz_d(); + } + + out: + if (TRAPF_USERMODE(tf)) { + while (td->td_flags & (TDF_ASTPENDING|TDF_NEEDRESCHED)) { + enable_intr(); + ast(tf); + disable_intr(); + } + } +} + +static u_int +ia64_ih_invalid(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + ia64_srlz_d(); + panic("invalid XIV: %u", xiv); + return (1); +} + +static u_int +ia64_ih_irq(struct thread *td, u_int xiv, struct trapframe *tf) { struct ia64_intr *i; struct intr_event *ie; /* our interrupt event */ - /* - * Find the interrupt thread for this vector. - */ - i = ia64_intrs[vector]; - KASSERT(i != NULL, ("%s: unassigned vector", __func__)); + PCPU_INC(md.stats.pcs_nhwints); + + /* Find the interrupt thread for this XIV. */ + i = ia64_intrs[xiv]; + KASSERT(i != NULL, ("%s: unassigned XIV", __func__)); (*i->cntp)++; ie = i->event; KASSERT(ie != NULL, ("%s: interrupt without event", __func__)); - if (intr_event_handle(ie, frame) != 0) { - /* - * XXX: The pre-INTR_FILTER code didn't mask stray - * interrupts. - */ - ia64_intr_mask((void *)(uintptr_t)vector); + if (intr_event_handle(ie, tf) != 0) { + ia64_intr_mask((void *)(uintptr_t)xiv); log(LOG_ERR, "stray irq%u\n", i->irq); } + ia64_set_eoi(0); + ia64_srlz_d(); + return (0); } #ifdef DDB static void -db_print_vector(u_int vector, int always) +db_print_xiv(u_int xiv, int always) { struct ia64_intr *i; - i = ia64_intrs[vector]; + i = ia64_intrs[xiv]; if (i != NULL) { - db_printf("vector %u (%p): ", vector, i); + db_printf("XIV %u (%p): ", xiv, i); sapic_print(i->sapic, i->irq); } else if (always) - db_printf("vector %u: unassigned\n", vector); + db_printf("XIV %u: unassigned\n", xiv); } -DB_SHOW_COMMAND(vector, db_show_vector) +DB_SHOW_COMMAND(xiv, db_show_xiv) { - u_int vector; + u_int xiv; if (have_addr) { - vector = ((addr >> 4) % 16) * 10 + (addr % 16); - if (vector >= 256) - db_printf("error: vector %u not in range [0..255]\n", - vector); + xiv = ((addr >> 4) % 16) * 10 + (addr % 16); + if (xiv >= IA64_NXIVS) + db_printf("error: XIV %u not in range [0..%u]\n", + xiv, IA64_NXIVS - 1); else - db_print_vector(vector, 1); + db_print_xiv(xiv, 1); } else { - for (vector = 0; vector < 256; vector++) - db_print_vector(vector, 0); + for (xiv = 0; xiv < IA64_NXIVS; xiv++) + db_print_xiv(xiv, 0); } } diff --git a/sys/ia64/ia64/locore.S b/sys/ia64/ia64/locore.S index afba834823b0..ce66dcacec6f 100644 --- a/sys/ia64/ia64/locore.S +++ b/sys/ia64/ia64/locore.S @@ -26,16 +26,13 @@ * $FreeBSD$ */ +#include #include #include -#include -#include -#include - -#ifndef EVCNT_COUNTERS -#define _LOCORE #include -#endif +#include +#include +#include .section .data.proc0,"aw" .global kstack @@ -310,7 +307,7 @@ EXPORT(intrnames) .ascii "clock" .fill INTRNAME_LEN - 5 - 1, 1, ' ' .byte 0 -intr_n = 0 +intr_n = 1 .rept INTRCNT_COUNT - 1 .ascii "#" .byte intr_n / 100 + '0' diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index ed7a288e0c24..fd1bc2aea878 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -371,7 +371,7 @@ cpu_startup(void *dummy) SYSCTL_ADD_ULONG(&pc->pc_md.sysctl_ctx, SYSCTL_CHILDREN(pc->pc_md.sysctl_tree), OID_AUTO, "nstrays", CTLFLAG_RD, &pcs->pcs_nstrays, - "Number of stray vectors"); + "Number of stray interrupts"); } } SYSINIT(cpu_startup, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); @@ -781,6 +781,7 @@ ia64_init(void) */ map_pal_code(); efi_boot_minimal(bootinfo.bi_systab); + ia64_xiv_init(); ia64_sal_init(); calculate_frequencies(); diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c index c5ed48f501d6..b1d32f706c1a 100644 --- a/sys/ia64/ia64/mp_machdep.c +++ b/sys/ia64/ia64/mp_machdep.c @@ -46,11 +46,6 @@ __FBSDID("$FreeBSD$"); #include #include -#include -#include -#include -#include - #include #include #include @@ -59,10 +54,13 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include #include #include -#include + +#include +#include +#include +#include MALLOC_DEFINE(M_SMP, "SMP", "SMP related allocations"); @@ -81,7 +79,78 @@ volatile int ap_delay; volatile int ap_awake; volatile int ap_spin; -static void cpu_mp_unleash(void *); +int ia64_ipi_ast; +int ia64_ipi_highfp; +int ia64_ipi_nmi; +int ia64_ipi_preempt; +int ia64_ipi_rndzvs; +int ia64_ipi_stop; + +static u_int +ia64_ih_ast(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nasts); + CTR1(KTR_SMP, "IPI_AST, cpuid=%d", PCPU_GET(cpuid)); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_highfp(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nhighfps); + ia64_highfp_save_ipi(); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_preempt(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_npreempts); + CTR1(KTR_SMP, "IPI_PREEMPT, cpuid=%d", PCPU_GET(cpuid)); + sched_preempt(curthread); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_rndzvs(struct thread *td, u_int xiv, struct trapframe *tf) +{ + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nrdvs); + CTR1(KTR_SMP, "IPI_RENDEZVOUS, cpuid=%d", PCPU_GET(cpuid)); + smp_rendezvous_action(); + ia64_srlz_d(); + return (0); +} + +static u_int +ia64_ih_stop(struct thread *td, u_int xiv, struct trapframe *tf) +{ + cpumask_t mybit; + + ia64_set_eoi(0); + PCPU_INC(md.stats.pcs_nstops); + mybit = PCPU_GET(cpumask); + ia64_srlz_d(); + + savectx(PCPU_PTR(md.pcb)); + + atomic_set_int(&stopped_cpus, mybit); + while ((started_cpus & mybit) == 0) + cpu_spinwait(); + atomic_clear_int(&started_cpus, mybit); + atomic_clear_int(&stopped_cpus, mybit); + return (0); +} struct cpu_group * cpu_topo(void) @@ -116,7 +185,6 @@ void ia64_ap_startup(void) { uint64_t vhpt; - int vector; pcpup = ap_pcpu; ia64_set_k4((intptr_t)pcpup); @@ -148,18 +216,6 @@ ia64_ap_startup(void) CTR1(KTR_SMP, "SMP: cpu%d launched", PCPU_GET(cpuid)); - /* Acknowledge and EOI all interrupts. */ - vector = ia64_get_ivr(); - while (vector != 15) { - ia64_srlz_d(); - if (vector == 0) - vector = (int)ia64_ld1(&ia64_pib->ib_inta); - ia64_set_eoi(0); - ia64_srlz_d(); - vector = ia64_get_ivr(); - } - ia64_srlz_d(); - /* kick off the clock on this AP */ pcpu_initclock(); @@ -200,7 +256,7 @@ cpu_mp_probe(void) * case we can have multiple processors, but we simply can't wake * them up... */ - return (mp_ncpus > 1 && ipi_vector[IPI_AP_WAKEUP] != 0); + return (mp_ncpus > 1 && ia64_ipi_wakeup != 0); } void @@ -276,7 +332,7 @@ cpu_mp_start() if (bootverbose) printf("SMP: waking up cpu%d\n", pc->pc_cpuid); - ipi_send(pc, IPI_AP_WAKEUP); + ipi_send(pc, ia64_ipi_wakeup); do { DELAY(1000); @@ -300,6 +356,18 @@ cpu_mp_unleash(void *dummy) if (mp_ncpus <= 1) return; + /* Allocate XIVs for IPIs */ + ia64_ipi_ast = ia64_xiv_alloc(PI_DULL, IA64_XIV_IPI, ia64_ih_ast); + ia64_ipi_highfp = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_highfp); + ia64_ipi_preempt = ia64_xiv_alloc(PI_SOFT, IA64_XIV_IPI, + ia64_ih_preempt); + ia64_ipi_rndzvs = ia64_xiv_alloc(PI_AV, IA64_XIV_IPI, ia64_ih_rndzvs); + ia64_ipi_stop = ia64_xiv_alloc(PI_REALTIME, IA64_XIV_IPI, ia64_ih_stop); + + /* Reserve the NMI vector for IPI_STOP_HARD if possible */ + ia64_ipi_nmi = (ia64_xiv_reserve(2, IA64_XIV_IPI, ia64_ih_stop) != 0) + ? ia64_ipi_stop : 0x400; /* DM=NMI, Vector=n/a */ + cpus = 0; smp_cpus = 0; SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { @@ -361,20 +429,18 @@ ipi_all_but_self(int ipi) * fields are used here. */ void -ipi_send(struct pcpu *cpu, int ipi) +ipi_send(struct pcpu *cpu, int xiv) { u_int lid; - uint8_t vector; + + KASSERT(xiv != 0, ("ipi_send")); lid = LID_SAPIC(cpu->pc_md.lid); - vector = ipi_vector[ipi]; - KASSERT(vector != 0, ("IPI %d is not assigned a vector", ipi)); ia64_mf(); - ia64_st8(&(ia64_pib->ib_ipi[lid][0]), vector); + ia64_st8(&(ia64_pib->ib_ipi[lid][0]), xiv); ia64_mf_a(); - CTR4(KTR_SMP, "ipi_send(%p, %ld): cpuid=%d, vector=%u", cpu, ipi, - PCPU_GET(cpuid), vector); + CTR3(KTR_SMP, "ipi_send(%p, %d): cpuid=%d", cpu, xiv, PCPU_GET(cpuid)); } SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); diff --git a/sys/ia64/ia64/nexus.c b/sys/ia64/ia64/nexus.c index 2f7c3744a2be..69f62a9b2fbd 100644 --- a/sys/ia64/ia64/nexus.c +++ b/sys/ia64/ia64/nexus.c @@ -159,41 +159,15 @@ nexus_probe(device_t dev) device_quiet(dev); /* suppress attach message for neatness */ - /* - * XXX working notes: - * - * - IRQ resource creation should be moved to the PIC/APIC driver. - * - DRQ resource creation should be moved to the DMAC driver. - * - The above should be sorted to probe earlier than any child busses. - * - * - Leave I/O and memory creation here, as child probes may need them. - * (especially eg. ACPI) - */ - - /* - * IRQ's are on the mainboard on old systems, but on the ISA part - * of PCI->ISA bridges. There would be multiple sets of IRQs on - * multi-ISA-bus systems. PCI interrupts are routed to the ISA - * component, so in a way, PCI can be a partial child of an ISA bus(!). - * APIC interrupts are global though. - * - * XXX We depend on the AT PIC driver correctly claiming IRQ 2 - * to prevent its reuse elsewhere in the !APIC_IO case. - */ - irq_rman.rm_start = 0; irq_rman.rm_type = RMAN_ARRAY; irq_rman.rm_descr = "Interrupt request lines"; - irq_rman.rm_end = 255; + irq_rman.rm_start = 0; + irq_rman.rm_end = IA64_NXIVS - 1; if (rman_init(&irq_rman) || rman_manage_region(&irq_rman, irq_rman.rm_start, irq_rman.rm_end)) panic("nexus_probe irq_rman"); - /* - * However, IO ports and Memory truely are global at this level, - * as are APIC interrupts (however many IO APICS there turn out - * to be on large systems..) - */ port_rman.rm_start = 0; port_rman.rm_end = 0xffff; port_rman.rm_type = RMAN_ARRAY; diff --git a/sys/ia64/ia64/sal.c b/sys/ia64/ia64/sal.c index 4f46d652b02a..6f081be8601a 100644 --- a/sys/ia64/ia64/sal.c +++ b/sys/ia64/ia64/sal.c @@ -29,21 +29,18 @@ __FBSDID("$FreeBSD$"); #include #include +#include #include #include #include #include #include +#include #include #include #include -/* - * IPIs are used more genericly than only - * for inter-processor interrupts. Don't - * make it a SMP specific thing... - */ -int ipi_vector[IPI_COUNT]; +int ia64_ipi_wakeup; static struct ia64_fdesc sal_fdesc; static sal_entry_t fake_sal; @@ -66,22 +63,6 @@ fake_sal(u_int64_t a1, u_int64_t a2, u_int64_t a3, u_int64_t a4, return res; } -static void -setup_ipi_vectors(int ceil) -{ - int ipi; - - ipi_vector[IPI_MCA_RENDEZ] = ceil - 0x10; - - ipi = IPI_AST; /* First generic IPI. */ - ceil -= 0x20; /* First vector in group. */ - while (ipi < IPI_COUNT) - ipi_vector[ipi++] = ceil++; - - ipi_vector[IPI_HIGH_FP] = ceil - 0x30; - ipi_vector[IPI_MCA_CMCV] = ceil - 0x30 + 1; -} - void ia64_sal_init(void) { @@ -89,7 +70,7 @@ ia64_sal_init(void) 48, 32, 16, 32, 16, 16 }; u_int8_t *p; - int i; + int error, i; sal_systbl = efi_get_table(&sal_table); if (sal_systbl == NULL) @@ -132,36 +113,19 @@ ia64_sal_init(void) break; } - if (dp->sale_vector < 0x10 || dp->sale_vector > 0xff) { - printf("SAL: invalid AP wake-up vector " - "(0x%lx)\n", dp->sale_vector); - break; - } - - /* - * SAL documents that the wake-up vector should be - * high (close to 255). The MCA rendezvous vector - * should be less than the wake-up vector, but still - * "high". We use the following priority assignment: - * Wake-up: priority of the sale_vector - * Rendezvous: priority-1 - * Generic IPIs: priority-2 - * Special IPIs: priority-3 - * Consequently, the wake-up priority should be at - * least 4 (ie vector >= 0x40). - */ - if (dp->sale_vector < 0x40) { - printf("SAL: AP wake-up vector too low " - "(0x%lx)\n", dp->sale_vector); - break; - } - - if (bootverbose) - printf("SAL: AP wake-up vector: 0x%lx\n", + /* Reserve the XIV so that we won't use it. */ + error = ia64_xiv_reserve(dp->sale_vector, + IA64_XIV_PLAT, NULL); + if (error) { + printf("SAL: invalid AP wake-up XIV (%#lx)\n", dp->sale_vector); + break; + } - ipi_vector[IPI_AP_WAKEUP] = dp->sale_vector; - setup_ipi_vectors(dp->sale_vector & 0xf0); + ia64_ipi_wakeup = dp->sale_vector; + if (bootverbose) + printf("SAL: AP wake-up XIV: %#x\n", + ia64_ipi_wakeup); #ifdef SMP fd = (struct ia64_fdesc *) os_boot_rendez; @@ -175,7 +139,4 @@ ia64_sal_init(void) } p += sizes[*p]; } - - if (ipi_vector[IPI_AP_WAKEUP] == 0) - setup_ipi_vectors(0xf0); } diff --git a/sys/ia64/include/clock.h b/sys/ia64/include/clock.h index 772fc2a29ed3..6b87a89f9ede 100644 --- a/sys/ia64/include/clock.h +++ b/sys/ia64/include/clock.h @@ -9,12 +9,4 @@ #ifndef _MACHINE_CLOCK_H_ #define _MACHINE_CLOCK_H_ -#ifdef _KERNEL - -#define CLOCK_VECTOR 254 - -extern uint64_t ia64_clock_reload; - -#endif - #endif /* !_MACHINE_CLOCK_H_ */ diff --git a/sys/ia64/include/intr.h b/sys/ia64/include/intr.h index 0635e3409f53..b26190c52fdd 100644 --- a/sys/ia64/include/intr.h +++ b/sys/ia64/include/intr.h @@ -30,7 +30,14 @@ #ifndef _MACHINE_INTR_H_ #define _MACHINE_INTR_H_ +#define IA64_NXIVS 256 /* External Interrupt Vectors */ +#define IA64_MIN_XIV 16 + +#define IA64_MAX_HWPRIO 14 + struct sapic; +struct thread; +struct trapframe; /* * Layout of the Processor Interrupt Block. @@ -46,12 +53,28 @@ struct ia64_pib uint8_t _rsvd4[0x1fff0]; }; +enum ia64_xiv_use { + IA64_XIV_FREE, + IA64_XIV_ARCH, /* Architecturally defined. */ + IA64_XIV_PLAT, /* Platform defined. */ + IA64_XIV_IPI, /* Used for IPIs. */ + IA64_XIV_IRQ /* Used for external interrupts. */ +}; + +typedef u_int (ia64_ihtype)(struct thread *, u_int, struct trapframe *); + extern struct ia64_pib *ia64_pib; +void ia64_handle_intr(struct trapframe *); int ia64_setup_intr(const char *, int, driver_filter_t, driver_intr_t, void *, enum intr_type, void **); int ia64_teardown_intr(void *); +void ia64_xiv_init(void); +u_int ia64_xiv_alloc(u_int, enum ia64_xiv_use, ia64_ihtype); +int ia64_xiv_free(u_int, enum ia64_xiv_use); +int ia64_xiv_reserve(u_int, enum ia64_xiv_use, ia64_ihtype); + int sapic_config_intr(u_int, enum intr_trigger, enum intr_polarity); struct sapic *sapic_create(u_int, u_int, uint64_t); int sapic_enable(struct sapic *, u_int, u_int); diff --git a/sys/ia64/include/intrcnt.h b/sys/ia64/include/intrcnt.h index c4f73c014306..5e165ea75efe 100644 --- a/sys/ia64/include/intrcnt.h +++ b/sys/ia64/include/intrcnt.h @@ -29,11 +29,7 @@ */ #define INTRCNT_CLOCK 0 -#define INTRCNT_ISA_IRQ (INTRCNT_CLOCK + 1) -#define INTRCNT_ISA_IRQ_LEN 16 -#define INTRCNT_OTHER_BASE (INTRCNT_ISA_IRQ + INTRCNT_ISA_IRQ_LEN) -#define INTRCNT_OTHER_LEN 240 -#define INTRCNT_COUNT (INTRCNT_OTHER_BASE + INTRCNT_OTHER_LEN) +#define INTRCNT_COUNT 256 /* * Maximum name length in intrnames table (including terminating '\0'. diff --git a/sys/ia64/include/smp.h b/sys/ia64/include/smp.h index 4eddf7434fea..9f976dec5970 100644 --- a/sys/ia64/include/smp.h +++ b/sys/ia64/include/smp.h @@ -6,31 +6,23 @@ #ifdef _KERNEL -/* - * Interprocessor interrupts for SMP. The following values are indices - * into the IPI vector table. The SAL gives us the vector used for AP - * wake-up. We base the other vectors on that. Keep IPI_AP_WAKEUP at - * index 0. See sal.c for details. - */ -/* Architecture specific IPIs. */ -#define IPI_AP_WAKEUP 0 -#define IPI_HIGH_FP 1 -#define IPI_MCA_CMCV 2 -#define IPI_MCA_RENDEZ 3 -/* Machine independent IPIs. */ -#define IPI_AST 4 -#define IPI_RENDEZVOUS 5 -#define IPI_STOP 6 -#define IPI_STOP_HARD 6 -#define IPI_PREEMPT 7 - -#define IPI_COUNT 8 +#define IPI_AST ia64_ipi_ast +#define IPI_PREEMPT ia64_ipi_preempt +#define IPI_RENDEZVOUS ia64_ipi_rndzvs +#define IPI_STOP ia64_ipi_stop +#define IPI_STOP_HARD ia64_ipi_nmi #ifndef LOCORE struct pcpu; -extern int ipi_vector[]; +extern int ia64_ipi_ast; +extern int ia64_ipi_highfp; +extern int ia64_ipi_nmi; +extern int ia64_ipi_preempt; +extern int ia64_ipi_rndzvs; +extern int ia64_ipi_stop; +extern int ia64_ipi_wakeup; void ipi_all_but_self(int ipi); void ipi_selected(cpumask_t cpus, int ipi);