diff --git a/sys/arm64/arm64/gic.c b/sys/arm64/arm64/gic.c deleted file mode 100644 index 68b7caba58d5..000000000000 --- a/sys/arm64/arm64/gic.c +++ /dev/null @@ -1,496 +0,0 @@ -/*- - * Copyright (c) 2011 The FreeBSD Foundation - * Copyright (c) 2014 Andrew Turner - * All rights reserved. - * - * Developed by Damjan Marion - * - * Based on OMAP4 GIC code by Ben Gray - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the company nor the name of the author may be used to - * endorse or promote products derived from this software without specific - * prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -#include - -#include "pic_if.h" - -/* We are using GICv2 register naming */ - -/* Distributor Registers */ -#define GICD_CTLR 0x000 /* v1 ICDDCR */ -#define GICD_TYPER 0x004 /* v1 ICDICTR */ -#define GICD_IIDR 0x008 /* v1 ICDIIDR */ -#define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ -#define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ -#define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ -#define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ -#define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ -#define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ -#define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ -#define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ -#define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ -#define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ -#define GICD_SGI_TARGET_SHIFT 16 - -/* CPU Registers */ -#define GICC_CTLR 0x0000 /* v1 ICCICR */ -#define GICC_PMR 0x0004 /* v1 ICCPMR */ -#define GICC_BPR 0x0008 /* v1 ICCBPR */ -#define GICC_IAR 0x000C /* v1 ICCIAR */ -#define GICC_EOIR 0x0010 /* v1 ICCEOIR */ -#define GICC_RPR 0x0014 /* v1 ICCRPR */ -#define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ -#define GICC_ABPR 0x001C /* v1 ICCABPR */ -#define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ - -#define GIC_FIRST_IPI 0 /* Irqs 0-15 are SGIs/IPIs. */ -#define GIC_LAST_IPI 15 -#define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ -#define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ -#define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ - -/* TYPER Registers */ -#define GICD_TYPER_SECURITYEXT 0x400 -#define GIC_SUPPORT_SECEXT(_sc) \ - ((_sc->typer & GICD_TYPER_SECURITYEXT) == GICD_TYPER_SECURITYEXT) - -/* First bit is a polarity bit (0 - low, 1 - high) */ -#define GICD_ICFGR_POL_LOW (0 << 0) -#define GICD_ICFGR_POL_HIGH (1 << 0) -#define GICD_ICFGR_POL_MASK 0x1 -/* Second bit is a trigger bit (0 - level, 1 - edge) */ -#define GICD_ICFGR_TRIG_LVL (0 << 1) -#define GICD_ICFGR_TRIG_EDGE (1 << 1) -#define GICD_ICFGR_TRIG_MASK 0x2 - -static struct resource_spec arm_gic_spec[] = { - { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ - { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ - { -1, 0 } -}; - -static u_int arm_gic_map[MAXCPU]; - -static struct arm_gic_softc *arm_gic_sc = NULL; - -#define gic_c_read_4(_sc, _reg) \ - bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) -#define gic_c_write_4(_sc, _reg, _val) \ - bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) -#define gic_d_read_4(_sc, _reg) \ - bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) -#define gic_d_write_4(_sc, _reg, _val) \ - bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) - -static pic_dispatch_t gic_dispatch; -static pic_eoi_t gic_eoi; -static pic_mask_t gic_mask_irq; -static pic_unmask_t gic_unmask_irq; - -static uint8_t -gic_cpu_mask(struct arm_gic_softc *sc) -{ - uint32_t mask; - int i; - - /* Read the current cpuid mask by reading ITARGETSR{0..7} */ - for (i = 0; i < 8; i++) { - mask = gic_d_read_4(sc, GICD_ITARGETSR(i)); - if (mask != 0) - break; - } - /* No mask found, assume we are on CPU interface 0 */ - if (mask == 0) - return (1); - - /* Collect the mask in the lower byte */ - mask |= mask >> 16; - mask |= mask >> 8; - - return (mask); -} - -#ifdef SMP -static void -gic_init_secondary(device_t dev) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - int i; - - /* Set the mask so we can find this CPU to send it IPIs */ - arm_gic_map[PCPU_GET(cpuid)] = gic_cpu_mask(sc); - - for (i = 0; i < sc->nirqs; i += 4) - gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); - - /* Set all the interrupts to be in Group 0 (secure) */ - for (i = 0; GIC_SUPPORT_SECEXT(sc) && i < sc->nirqs; i += 32) { - gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); - } - - /* Enable CPU interface */ - gic_c_write_4(sc, GICC_CTLR, 1); - - /* Set priority mask register. */ - gic_c_write_4(sc, GICC_PMR, 0xff); - - /* Enable interrupt distribution */ - gic_d_write_4(sc, GICD_CTLR, 0x01); - - /* - * Activate the timer interrupts: virtual, secure, and non-secure. - */ - gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); - gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); - gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); -} -#endif - -int -arm_gic_attach(device_t dev) -{ - struct arm_gic_softc *sc; - int i; - uint32_t icciidr, mask; - - if (arm_gic_sc) - return (ENXIO); - - sc = device_get_softc(dev); - - if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { - device_printf(dev, "could not allocate resources\n"); - return (ENXIO); - } - - sc->gic_dev = dev; - arm_gic_sc = sc; - - /* Initialize mutex */ - mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); - - /* Distributor Interface */ - sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); - sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); - - /* CPU Interface */ - sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); - sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); - - /* Disable interrupt forwarding to the CPU interface */ - gic_d_write_4(sc, GICD_CTLR, 0x00); - - /* Get the number of interrupts */ - sc->typer = gic_d_read_4(sc, GICD_TYPER); - sc->nirqs = 32 * ((sc->typer & 0x1f) + 1); - - arm_register_root_pic(dev, sc->nirqs); - - icciidr = gic_c_read_4(sc, GICC_IIDR); - device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", - icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, - (icciidr & 0xfff), sc->nirqs); - - /* Set all global interrupts to be level triggered, active low. */ - for (i = 32; i < sc->nirqs; i += 16) { - gic_d_write_4(sc, GICD_ICFGR(i >> 4), 0x00000000); - } - - /* Disable all interrupts. */ - for (i = 32; i < sc->nirqs; i += 32) { - gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); - } - - /* Find the current cpu mask */ - mask = gic_cpu_mask(sc); - /* Set the mask so we can find this CPU to send it IPIs */ - arm_gic_map[PCPU_GET(cpuid)] = mask; - /* Set all four targets to this cpu */ - mask |= mask << 8; - mask |= mask << 16; - - for (i = 0; i < sc->nirqs; i += 4) { - gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); - if (i > 32) { - gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), mask); - } - } - - /* Set all the interrupts to be in Group 0 (secure) */ - for (i = 0; GIC_SUPPORT_SECEXT(sc) && i < sc->nirqs; i += 32) { - gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); - } - - /* Enable CPU interface */ - gic_c_write_4(sc, GICC_CTLR, 1); - - /* Set priority mask register. */ - gic_c_write_4(sc, GICC_PMR, 0xff); - - /* Enable interrupt distribution */ - gic_d_write_4(sc, GICD_CTLR, 0x01); - - return (0); -} - -static void gic_dispatch(device_t dev, struct trapframe *frame) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - uint32_t active_irq; - int first = 1; - - while (1) { - active_irq = gic_c_read_4(sc, GICC_IAR); - - /* - * Immediatly EOIR the SGIs, because doing so requires the other - * bits (ie CPU number), not just the IRQ number, and we do not - * have this information later. - */ - - if ((active_irq & 0x3ff) <= GIC_LAST_IPI) - gic_c_write_4(sc, GICC_EOIR, active_irq); - active_irq &= 0x3FF; - - if (active_irq == 0x3FF) { - if (first) - printf("Spurious interrupt detected\n"); - return; - } - - arm_dispatch_intr(active_irq, frame); - first = 0; - } -} - -static void -gic_eoi(device_t dev, u_int irq) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - - gic_c_write_4(sc, GICC_EOIR, irq); -} - -void -gic_mask_irq(device_t dev, u_int irq) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - - gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); - gic_c_write_4(sc, GICC_EOIR, irq); -} - -void -gic_unmask_irq(device_t dev, u_int irq) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - - gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); -} - -#ifdef SMP -static void -gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) -{ - struct arm_gic_softc *sc = device_get_softc(dev); - uint32_t val = 0, i; - - for (i = 0; i < MAXCPU; i++) - if (CPU_ISSET(i, &cpus)) - val |= arm_gic_map[i] << GICD_SGI_TARGET_SHIFT; - - gic_d_write_4(sc, GICD_SGIR(0), val | ipi); -} -#endif - -static device_method_t arm_gic_methods[] = { - /* Device interface */ - DEVMETHOD(device_attach, arm_gic_attach), - - /* pic_if */ - DEVMETHOD(pic_dispatch, gic_dispatch), - DEVMETHOD(pic_eoi, gic_eoi), - DEVMETHOD(pic_mask, gic_mask_irq), - DEVMETHOD(pic_unmask, gic_unmask_irq), - -#ifdef SMP - DEVMETHOD(pic_init_secondary, gic_init_secondary), - DEVMETHOD(pic_ipi_send, gic_ipi_send), -#endif - - { 0, 0 } -}; - -DEFINE_CLASS_0(gic, arm_gic_driver, arm_gic_methods, - sizeof(struct arm_gic_softc)); - -#define GICV2M_MSI_TYPER 0x008 -#define MSI_TYPER_SPI_BASE(x) (((x) >> 16) & 0x3ff) -#define MSI_TYPER_SPI_COUNT(x) (((x) >> 0) & 0x3ff) -#define GICv2M_MSI_SETSPI_NS 0x040 -#define GICV2M_MSI_IIDR 0xFCC - -static int -gicv2m_attach(device_t dev) -{ - struct gicv2m_softc *sc; - uint32_t typer; - int rid; - - sc = device_get_softc(dev); - - rid = 0; - sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); - if (sc->sc_mem == NULL) { - device_printf(dev, "Unable to allocate resources\n"); - return (ENXIO); - } - - typer = bus_read_4(sc->sc_mem, GICV2M_MSI_TYPER); - sc->sc_spi_start = MSI_TYPER_SPI_BASE(typer); - sc->sc_spi_count = MSI_TYPER_SPI_COUNT(typer); - - device_printf(dev, "using spi %u to %u\n", sc->sc_spi_start, - sc->sc_spi_start + sc->sc_spi_count - 1); - - mtx_init(&sc->sc_mutex, "GICv2m lock", "", MTX_DEF); - - arm_register_msi_pic(dev); - - return (0); -} - -static int -gicv2m_alloc_msix(device_t dev, device_t pci_dev, int *pirq) -{ - struct arm_gic_softc *psc; - struct gicv2m_softc *sc; - uint32_t reg; - int irq; - - psc = device_get_softc(device_get_parent(dev)); - sc = device_get_softc(dev); - - mtx_lock(&sc->sc_mutex); - /* Find an unused interrupt */ - KASSERT(sc->sc_spi_offset < sc->sc_spi_count, ("No free SPIs")); - - irq = sc->sc_spi_start + sc->sc_spi_offset; - sc->sc_spi_offset++; - - /* Interrupts need to be edge triggered, set this */ - reg = gic_d_read_4(psc, GICD_ICFGR(irq >> 4)); - reg |= (GICD_ICFGR_TRIG_EDGE | GICD_ICFGR_POL_HIGH) << - ((irq & 0xf) * 2); - gic_d_write_4(psc, GICD_ICFGR(irq >> 4), reg); - - *pirq = irq; - mtx_unlock(&sc->sc_mutex); - - return (0); -} - -static int -gicv2m_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs) -{ - struct arm_gic_softc *psc; - struct gicv2m_softc *sc; - uint32_t reg; - int i, irq; - - psc = device_get_softc(device_get_parent(dev)); - sc = device_get_softc(dev); - - mtx_lock(&sc->sc_mutex); - KASSERT(sc->sc_spi_offset + count <= sc->sc_spi_count, - ("No free SPIs for %d MSI interrupts", count)); - - /* Find an unused interrupt */ - for (i = 0; i < count; i++) { - irq = sc->sc_spi_start + sc->sc_spi_offset; - sc->sc_spi_offset++; - - /* Interrupts need to be edge triggered, set this */ - reg = gic_d_read_4(psc, GICD_ICFGR(irq >> 4)); - reg |= (GICD_ICFGR_TRIG_EDGE | GICD_ICFGR_POL_HIGH) << - ((irq & 0xf) * 2); - gic_d_write_4(psc, GICD_ICFGR(irq >> 4), reg); - - irqs[i] = irq; - } - mtx_unlock(&sc->sc_mutex); - - return (0); -} - -static int -gicv2m_map_msi(device_t dev, device_t pci_dev, int irq, uint64_t *addr, - uint32_t *data) -{ - struct gicv2m_softc *sc = device_get_softc(dev); - - *addr = vtophys(rman_get_virtual(sc->sc_mem)) + 0x40; - *data = irq; - - return (0); -} - -static device_method_t arm_gicv2m_methods[] = { - /* Device interface */ - DEVMETHOD(device_attach, gicv2m_attach), - - /* MSI/MSI-X */ - DEVMETHOD(pic_alloc_msix, gicv2m_alloc_msix), - DEVMETHOD(pic_alloc_msi, gicv2m_alloc_msi), - DEVMETHOD(pic_map_msi, gicv2m_map_msi), - - { 0, 0 } -}; - -DEFINE_CLASS_0(gicv2m, arm_gicv2m_driver, arm_gicv2m_methods, - sizeof(struct gicv2m_softc)); diff --git a/sys/arm64/arm64/gic_acpi.c b/sys/arm64/arm64/gic_acpi.c deleted file mode 100644 index ad26c0c43bcd..000000000000 --- a/sys/arm64/arm64/gic_acpi.c +++ /dev/null @@ -1,161 +0,0 @@ -/*- - * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Andrew Turner under - * sponsorship from the FreeBSD Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include - -#include - -#include - -#include - -#include - -struct arm_gic_acpi_softc { - struct arm_gic_softc gic_sc; - struct resource_list res; -}; - -struct madt_table_data { - device_t parent; - ACPI_MADT_GENERIC_DISTRIBUTOR *dist; - ACPI_MADT_GENERIC_INTERRUPT *intr; -}; - -static void -madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) -{ - struct madt_table_data *madt_data; - - madt_data = (struct madt_table_data *)arg; - - switch(entry->Type) { - case ACPI_MADT_TYPE_GENERIC_INTERRUPT: - if (madt_data->intr != NULL) { - if (bootverbose) - device_printf(madt_data->parent, - "gic: Already have an interrupt table"); - break; - } - - madt_data->intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; - break; - - case ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR: - if (madt_data->dist != NULL) { - if (bootverbose) - device_printf(madt_data->parent, - "gic: Already have a distributor table"); - break; - } - - madt_data->dist = (ACPI_MADT_GENERIC_DISTRIBUTOR *)entry; - break; - - default: - break; - } -} - -static void -arm_gic_acpi_identify(driver_t *driver, device_t parent) -{ - struct madt_table_data madt_data; - ACPI_TABLE_MADT *madt; - vm_paddr_t physaddr; - device_t dev; - - physaddr = acpi_find_table(ACPI_SIG_MADT); - if (physaddr == 0) - return; - - madt = acpi_map_table(physaddr, ACPI_SIG_MADT); - if (madt == NULL) { - device_printf(parent, "gic: Unable to map the MADT\n"); - return; - } - - madt_data.parent = parent; - madt_data.dist = NULL; - madt_data.intr = NULL; - - acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, - madt_handler, &madt_data); - if (madt_data.intr == NULL || madt_data.dist == NULL) { - device_printf(parent, - "No gic interrupt or distributor table\n"); - goto out; - } - - dev = BUS_ADD_CHILD(parent, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE, - "gic", -1); - if (dev == NULL) { - device_printf(parent, "add gic child failed\n"); - goto out; - } - - /* Add the MADT data */ - BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 0, - madt_data.dist->BaseAddress, PAGE_SIZE); - BUS_SET_RESOURCE(parent, dev, SYS_RES_MEMORY, 1, - madt_data.intr->BaseAddress, PAGE_SIZE); - -out: - acpi_unmap_table(madt); -} - -static int -arm_gic_acpi_probe(device_t dev) -{ - - device_set_desc(dev, "ARM Generic Interrupt Controller"); - return (BUS_PROBE_NOWILDCARD); -} - -static device_method_t arm_gic_acpi_methods[] = { - /* Device interface */ - DEVMETHOD(device_identify, arm_gic_acpi_identify), - DEVMETHOD(device_probe, arm_gic_acpi_probe), - - DEVMETHOD_END -}; - -DEFINE_CLASS_1(gic, arm_gic_acpi_driver, arm_gic_acpi_methods, - sizeof(struct arm_gic_acpi_softc), arm_gic_driver); - -static devclass_t arm_gic_acpi_devclass; - -EARLY_DRIVER_MODULE(gic, acpi, arm_gic_acpi_driver, - arm_gic_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/arm64/arm64/gic_fdt.c b/sys/arm64/arm64/gic_fdt.c deleted file mode 100644 index f92cd83c0ea6..000000000000 --- a/sys/arm64/arm64/gic_fdt.c +++ /dev/null @@ -1,332 +0,0 @@ -/*- - * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Andrew Turner under - * sponsorship from the FreeBSD Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include - -#include - -static struct ofw_compat_data compat_data[] = { - {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ - {"arm,gic-400", true}, - {"arm,cortex-a15-gic", true}, - {"arm,cortex-a9-gic", true}, - {"arm,cortex-a7-gic", true}, - {"arm,arm11mp-gic", true}, - {"brcm,brahma-b15-gic", true}, - {"qcom,msm-qgic2", true}, - {NULL, false} -}; - -struct gic_range { - uint64_t bus; - uint64_t host; - uint64_t size; -}; - -struct arm_gic_fdt_softc { - struct arm_gic_softc sc_gic; - pcell_t sc_host_cells; - pcell_t sc_addr_cells; - pcell_t sc_size_cells; - struct gic_range *sc_ranges; - int sc_nranges; -}; - -struct gic_devinfo { - struct ofw_bus_devinfo obdinfo; - struct resource_list rl; -}; - -static int -gic_fill_ranges(phandle_t node, struct arm_gic_fdt_softc *sc) -{ - cell_t *base_ranges; - ssize_t nbase_ranges; - int i, j, k; - - nbase_ranges = OF_getproplen(node, "ranges"); - if (nbase_ranges < 0) - return (-1); - sc->sc_nranges = nbase_ranges / sizeof(cell_t) / - (sc->sc_addr_cells + sc->sc_host_cells + sc->sc_size_cells); - if (sc->sc_nranges == 0) - return (0); - - sc->sc_ranges = malloc(sc->sc_nranges * sizeof(sc->sc_ranges[0]), - M_DEVBUF, M_WAITOK); - base_ranges = malloc(nbase_ranges, M_DEVBUF, M_WAITOK); - OF_getencprop(node, "ranges", base_ranges, nbase_ranges); - - for (i = 0, j = 0; i < sc->sc_nranges; i++) { - sc->sc_ranges[i].bus = 0; - for (k = 0; k < sc->sc_addr_cells; k++) { - sc->sc_ranges[i].bus <<= 32; - sc->sc_ranges[i].bus |= base_ranges[j++]; - } - sc->sc_ranges[i].host = 0; - for (k = 0; k < sc->sc_host_cells; k++) { - sc->sc_ranges[i].host <<= 32; - sc->sc_ranges[i].host |= base_ranges[j++]; - } - sc->sc_ranges[i].size = 0; - for (k = 0; k < sc->sc_size_cells; k++) { - sc->sc_ranges[i].size <<= 32; - sc->sc_ranges[i].size |= base_ranges[j++]; - } - } - - free(base_ranges, M_DEVBUF); - return (sc->sc_nranges); -} - -static int -arm_gic_fdt_probe(device_t dev) -{ - - if (!ofw_bus_status_okay(dev)) - return (ENXIO); - - if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) - return (ENXIO); - - device_set_desc(dev, "ARM Generic Interrupt Controller"); - return (BUS_PROBE_DEFAULT); -} - -static int -arm_gic_fdt_attach(device_t dev) -{ - struct arm_gic_fdt_softc *sc = device_get_softc(dev); - phandle_t root, child; - struct gic_devinfo *dinfo; - device_t cdev; - int err; - - err = arm_gic_attach(dev); - if (err != 0) - return (err); - - root = ofw_bus_get_node(dev); - - sc->sc_host_cells = 1; - OF_getencprop(OF_parent(root), "#address-cells", &sc->sc_host_cells, - sizeof(sc->sc_host_cells)); - sc->sc_addr_cells = 2; - OF_getencprop(root, "#address-cells", &sc->sc_addr_cells, - sizeof(sc->sc_addr_cells)); - sc->sc_size_cells = 2; - OF_getencprop(root, "#size-cells", &sc->sc_size_cells, - sizeof(sc->sc_size_cells)); - - /* If we have no children don't probe for them */ - child = OF_child(root); - if (child == 0) - return (0); - - if (gic_fill_ranges(root, sc) < 0) { - device_printf(dev, "could not get ranges\n"); - return (ENXIO); - } - - for (; child != 0; child = OF_peer(child)) { - dinfo = malloc(sizeof(*dinfo), M_DEVBUF, M_WAITOK | M_ZERO); - - if (ofw_bus_gen_setup_devinfo(&dinfo->obdinfo, child) != 0) { - free(dinfo, M_DEVBUF); - continue; - } - - resource_list_init(&dinfo->rl); - ofw_bus_reg_to_rl(dev, child, sc->sc_addr_cells, - sc->sc_size_cells, &dinfo->rl); - - cdev = device_add_child(dev, NULL, -1); - if (cdev == NULL) { - device_printf(dev, "<%s>: device_add_child failed\n", - dinfo->obdinfo.obd_name); - resource_list_free(&dinfo->rl); - ofw_bus_gen_destroy_devinfo(&dinfo->obdinfo); - free(dinfo, M_DEVBUF); - continue; - } - device_set_ivars(cdev, dinfo); - } - - bus_generic_probe(dev); - return (bus_generic_attach(dev)); -} - -static struct resource * -arm_gic_fdt_alloc_resource(device_t bus, device_t child, int type, int *rid, - rman_res_t start, rman_res_t end, rman_res_t count, u_int flags) -{ - struct arm_gic_fdt_softc *sc = device_get_softc(bus); - struct gic_devinfo *di; - struct resource_list_entry *rle; - int j; - - KASSERT(type == SYS_RES_MEMORY, ("Invalid resoure type %x", type)); - - /* - * Request for the default allocation with a given rid: use resource - * list stored in the local device info. - */ - if (RMAN_IS_DEFAULT_RANGE(start, end)) { - if ((di = device_get_ivars(child)) == NULL) - return (NULL); - - if (type == SYS_RES_IOPORT) - type = SYS_RES_MEMORY; - - rle = resource_list_find(&di->rl, type, *rid); - if (rle == NULL) { - if (bootverbose) - device_printf(bus, "no default resources for " - "rid = %d, type = %d\n", *rid, type); - return (NULL); - } - start = rle->start; - end = rle->end; - count = rle->count; - } - - /* Remap through ranges property */ - for (j = 0; j < sc->sc_nranges; j++) { - if (start >= sc->sc_ranges[j].bus && end < - sc->sc_ranges[j].bus + sc->sc_ranges[j].size) { - start -= sc->sc_ranges[j].bus; - start += sc->sc_ranges[j].host; - end -= sc->sc_ranges[j].bus; - end += sc->sc_ranges[j].host; - break; - } - } - if (j == sc->sc_nranges && sc->sc_nranges != 0) { - if (bootverbose) - device_printf(bus, "Could not map resource " - "%#lx-%#lx\n", start, end); - - return (NULL); - } - - return (bus_generic_alloc_resource(bus, child, type, rid, start, end, - count, flags)); -} - -static const struct ofw_bus_devinfo * -arm_gic_fdt_ofw_get_devinfo(device_t bus __unused, device_t child) -{ - struct gic_devinfo *di; - - di = device_get_ivars(child); - - return (&di->obdinfo); -} - - -static device_method_t arm_gic_fdt_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, arm_gic_fdt_probe), - DEVMETHOD(device_attach, arm_gic_fdt_attach), - - /* Bus interface */ - DEVMETHOD(bus_add_child, bus_generic_add_child), - DEVMETHOD(bus_alloc_resource, arm_gic_fdt_alloc_resource), - DEVMETHOD(bus_release_resource, bus_generic_release_resource), - DEVMETHOD(bus_activate_resource,bus_generic_activate_resource), - - /* ofw_bus interface */ - DEVMETHOD(ofw_bus_get_devinfo, arm_gic_fdt_ofw_get_devinfo), - DEVMETHOD(ofw_bus_get_compat, ofw_bus_gen_get_compat), - DEVMETHOD(ofw_bus_get_model, ofw_bus_gen_get_model), - DEVMETHOD(ofw_bus_get_name, ofw_bus_gen_get_name), - DEVMETHOD(ofw_bus_get_node, ofw_bus_gen_get_node), - DEVMETHOD(ofw_bus_get_type, ofw_bus_gen_get_type), - - DEVMETHOD_END -}; - -DEFINE_CLASS_1(gic, arm_gic_fdt_driver, arm_gic_fdt_methods, - sizeof(struct arm_gic_fdt_softc), arm_gic_driver); - -static devclass_t arm_gic_fdt_devclass; - -EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_fdt_driver, - arm_gic_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); -EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_fdt_driver, arm_gic_fdt_devclass, - 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); - -static struct ofw_compat_data gicv2m_compat_data[] = { - {"arm,gic-v2m-frame", true}, - {NULL, false} -}; - -static int -arm_gicv2m_fdt_probe(device_t dev) -{ - - if (!ofw_bus_status_okay(dev)) - return (ENXIO); - - if (!ofw_bus_search_compatible(dev, gicv2m_compat_data)->ocd_data) - return (ENXIO); - - device_set_desc(dev, "ARM Generic Interrupt Controller MSI/MSIX"); - return (BUS_PROBE_DEFAULT); -} - -static device_method_t arm_gicv2m_fdt_methods[] = { - /* Device interface */ - DEVMETHOD(device_probe, arm_gicv2m_fdt_probe), - - /* End */ - DEVMETHOD_END -}; - -DEFINE_CLASS_1(gicv2m, arm_gicv2m_fdt_driver, arm_gicv2m_fdt_methods, - sizeof(struct gicv2m_softc), arm_gicv2m_driver); - -static devclass_t arm_gicv2m_fdt_devclass; - -EARLY_DRIVER_MODULE(gicv2m, gic, arm_gicv2m_fdt_driver, - arm_gicv2m_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); diff --git a/sys/arm64/arm64/gic_v3_its.c b/sys/arm64/arm64/gic_v3_its.c deleted file mode 100644 index c16cf0a5befc..000000000000 --- a/sys/arm64/arm64/gic_v3_its.c +++ /dev/null @@ -1,1805 +0,0 @@ -/*- - * Copyright (c) 2015 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Semihalf under - * the sponsorship of the FreeBSD Foundation. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include "gic_v3_reg.h" -#include "gic_v3_var.h" - -#define GIC_V3_ITS_QUIRK_THUNDERX_PEM_BUS_OFFSET 88 - -#include "pic_if.h" -#include "pcib_if.h" - -/* Device and PIC methods */ -static int gic_v3_its_attach(device_t); - -static device_method_t gic_v3_its_methods[] = { - /* Device interface */ - DEVMETHOD(device_attach, gic_v3_its_attach), - /* - * PIC interface - */ - /* MSI-X */ - DEVMETHOD(pic_alloc_msix, gic_v3_its_alloc_msix), - DEVMETHOD(pic_release_msix, gic_v3_its_release_msix), - /* MSI */ - DEVMETHOD(pic_alloc_msi, gic_v3_its_alloc_msi), - DEVMETHOD(pic_release_msi, gic_v3_its_release_msi), - DEVMETHOD(pic_map_msi, gic_v3_its_map_msi), - - /* End */ - DEVMETHOD_END -}; - -DEFINE_CLASS_0(its, gic_v3_its_driver, gic_v3_its_methods, - sizeof(struct gic_v3_its_softc)); - -MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", GIC_V3_ITS_DEVSTR); - -static int its_alloc_tables(struct gic_v3_its_softc *); -static void its_free_tables(struct gic_v3_its_softc *); -static void its_init_commandq(struct gic_v3_its_softc *); -static void its_init_cpu_collection(struct gic_v3_its_softc *); -static uint32_t its_get_devid(device_t); -static struct its_dev * its_device_find_locked(struct gic_v3_its_softc *, - device_t, uint32_t); - -static int its_cmd_send(struct gic_v3_its_softc *, struct its_cmd_desc *); - -static void its_cmd_movi(struct gic_v3_its_softc *, struct its_dev *, - struct its_col *, uint32_t); -static void its_cmd_mapc(struct gic_v3_its_softc *, struct its_col *, uint8_t); -static void its_cmd_mapvi(struct gic_v3_its_softc *, struct its_dev *, uint32_t, - uint32_t); -static void its_cmd_mapi(struct gic_v3_its_softc *, struct its_dev *, uint32_t); -static void its_cmd_inv(struct gic_v3_its_softc *, struct its_dev *, uint32_t); -static void its_cmd_invall(struct gic_v3_its_softc *, struct its_col *); - -static uint32_t its_get_devbits(device_t); - -static void lpi_init_conftable(struct gic_v3_its_softc *); -static void lpi_bitmap_init(struct gic_v3_its_softc *); -static int lpi_config_cpu(struct gic_v3_its_softc *); -static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *); - -const char *its_ptab_cache[] = { - [GITS_BASER_CACHE_NCNB] = "(NC,NB)", - [GITS_BASER_CACHE_NC] = "(NC)", - [GITS_BASER_CACHE_RAWT] = "(RA,WT)", - [GITS_BASER_CACHE_RAWB] = "(RA,WB)", - [GITS_BASER_CACHE_WAWT] = "(WA,WT)", - [GITS_BASER_CACHE_WAWB] = "(WA,WB)", - [GITS_BASER_CACHE_RAWAWT] = "(RAWA,WT)", - [GITS_BASER_CACHE_RAWAWB] = "(RAWA,WB)", -}; - -const char *its_ptab_share[] = { - [GITS_BASER_SHARE_NS] = "none", - [GITS_BASER_SHARE_IS] = "inner", - [GITS_BASER_SHARE_OS] = "outer", - [GITS_BASER_SHARE_RES] = "none", -}; - -const char *its_ptab_type[] = { - [GITS_BASER_TYPE_UNIMPL] = "Unimplemented", - [GITS_BASER_TYPE_DEV] = "Devices", - [GITS_BASER_TYPE_VP] = "Virtual Processors", - [GITS_BASER_TYPE_PP] = "Physical Processors", - [GITS_BASER_TYPE_IC] = "Interrupt Collections", - [GITS_BASER_TYPE_RES5] = "Reserved (5)", - [GITS_BASER_TYPE_RES6] = "Reserved (6)", - [GITS_BASER_TYPE_RES7] = "Reserved (7)", -}; - -/* - * Vendor specific quirks. - * One needs to add appropriate entry to its_quirks[] - * table if the imlementation varies from the generic ARM ITS. - */ - -/* Cavium ThunderX PCI devid acquire function */ -static uint32_t its_get_devbits_thunder(device_t); - -static const struct its_quirks its_quirks[] = { - { - /* - * Hardware: Cavium ThunderX - * Chip revision: Pass 1.0, Pass 1.1 - */ - .cpuid = CPU_ID_RAW(CPU_IMPL_CAVIUM, CPU_PART_THUNDER, 0, 0), - .cpuid_mask = CPU_IMPL_MASK | CPU_PART_MASK, - .devbits_func = its_get_devbits_thunder, - }, -}; - -static struct gic_v3_its_softc *its_sc; - -#define gic_its_read(sc, len, reg) \ - bus_read_##len(&sc->its_res[0], reg) - -#define gic_its_write(sc, len, reg, val) \ - bus_write_##len(&sc->its_res[0], reg, val) - -static int -gic_v3_its_attach(device_t dev) -{ - struct gic_v3_its_softc *sc; - uint64_t gits_tmp; - uint32_t gits_pidr2; - int rid; - int ret; - - sc = device_get_softc(dev); - - /* - * XXX ARM64TODO: Avoid configuration of more than one ITS - * device. To be removed when multi-PIC support is added - * to FreeBSD (or at least multi-ITS is implemented). Limit - * supported ITS sockets to '0' only. - */ - if (device_get_unit(dev) != 0) { - device_printf(dev, - "Only single instance of ITS is supported, exiting...\n"); - return (ENXIO); - } - sc->its_socket = 0; - - /* - * Initialize sleep & spin mutex for ITS - */ - /* Protects ITS device list and assigned LPIs bitmaps. */ - mtx_init(&sc->its_dev_lock, "ITS dev lock", NULL, MTX_SPIN); - /* Protects access to ITS command circular buffer. */ - mtx_init(&sc->its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); - - rid = 0; - sc->its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, - RF_ACTIVE); - if (sc->its_res == NULL) { - device_printf(dev, "Could not allocate memory\n"); - return (ENXIO); - } - - sc->dev = dev; - - gits_pidr2 = gic_its_read(sc, 4, GITS_PIDR2); - switch (gits_pidr2 & GITS_PIDR2_ARCH_MASK) { - case GITS_PIDR2_ARCH_GICv3: /* fall through */ - case GITS_PIDR2_ARCH_GICv4: - if (bootverbose) { - device_printf(dev, "ITS found. Architecture rev. %u\n", - (u_int)(gits_pidr2 & GITS_PIDR2_ARCH_MASK) >> 4); - } - break; - default: - device_printf(dev, "No ITS found in the system\n"); - gic_v3_its_detach(dev); - return (ENODEV); - } - - /* 1. Initialize commands queue */ - its_init_commandq(sc); - - /* 2. Provide memory for any private ITS tables */ - ret = its_alloc_tables(sc); - if (ret != 0) { - gic_v3_its_detach(dev); - return (ret); - } - - /* 3. Allocate collections. One per-CPU */ - for (int cpu = 0; cpu < mp_ncpus; cpu++) - if (CPU_ISSET(cpu, &all_cpus) != 0) - sc->its_cols[cpu] = malloc(sizeof(*sc->its_cols[0]), - M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); - else - sc->its_cols[cpu] = NULL; - - /* 4. Enable ITS in GITS_CTLR */ - gits_tmp = gic_its_read(sc, 4, GITS_CTLR); - gic_its_write(sc, 4, GITS_CTLR, gits_tmp | GITS_CTLR_EN); - - /* 5. Initialize LPIs configuration table */ - lpi_init_conftable(sc); - - /* 6. LPIs bitmap init */ - lpi_bitmap_init(sc); - - /* 7. Allocate pending tables for all CPUs */ - lpi_alloc_cpu_pendtables(sc); - - /* 8. CPU init */ - (void)its_init_cpu(sc); - - /* 9. Init ITS devices list */ - TAILQ_INIT(&sc->its_dev_list); - - arm_register_msi_pic(dev); - - /* - * XXX ARM64TODO: We need to have ITS software context - * when being called by the interrupt code (mask/unmask). - * This may be used only when one ITS is present in - * the system and eventually should be removed. - */ - KASSERT(its_sc == NULL, - ("Trying to assign its_sc that is already set")); - its_sc = sc; - - return (0); -} - -/* Will not detach but use it for convenience */ -int -gic_v3_its_detach(device_t dev) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - struct gic_v3_its_softc *sc; - u_int cpuid; - int rid = 0; - - sc = device_get_softc(dev); - cpuid = PCPU_GET(cpuid); - - /* Release what's possible */ - - /* Command queue */ - if ((void *)sc->its_cmdq_base != NULL) { - contigfree((void *)sc->its_cmdq_base, - ITS_CMDQ_SIZE, M_GIC_V3_ITS); - } - /* ITTs */ - its_free_tables(sc); - /* Collections */ - for (cpuid = 0; cpuid < mp_ncpus; cpuid++) - free(sc->its_cols[cpuid], M_GIC_V3_ITS); - /* LPI config table */ - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - if ((void *)gic_sc->gic_redists.lpis.conf_base != NULL) { - contigfree((void *)gic_sc->gic_redists.lpis.conf_base, - LPI_CONFTAB_SIZE, M_GIC_V3_ITS); - } - for (cpuid = 0; cpuid < mp_ncpus; cpuid++) - if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { - contigfree( - (void *)gic_sc->gic_redists.lpis.pend_base[cpuid], - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), - M_GIC_V3_ITS); - } - - /* Resource... */ - bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); - - /* XXX ARM64TODO: Reset global pointer to ITS software context */ - its_sc = NULL; - - return (0); -} - -static int -its_alloc_tables(struct gic_v3_its_softc *sc) -{ - uint64_t gits_baser, gits_tmp; - uint64_t type, esize, cache, share, psz; - size_t page_size, npages, nitspages, nidents, tn; - size_t its_tbl_size; - vm_offset_t ptab_vaddr; - vm_paddr_t ptab_paddr; - boolean_t first = TRUE; - - page_size = PAGE_SIZE_64K; - - for (tn = 0; tn < GITS_BASER_NUM; tn++) { - gits_baser = gic_its_read(sc, 8, GITS_BASER(tn)); - type = GITS_BASER_TYPE(gits_baser); - /* Get the Table Entry size */ - esize = GITS_BASER_ESIZE(gits_baser); - - switch (type) { - case GITS_BASER_TYPE_UNIMPL: /* fall through */ - case GITS_BASER_TYPE_RES5: - case GITS_BASER_TYPE_RES6: - case GITS_BASER_TYPE_RES7: - continue; - case GITS_BASER_TYPE_DEV: - nidents = (1 << its_get_devbits(sc->dev)); - its_tbl_size = esize * nidents; - its_tbl_size = roundup2(its_tbl_size, page_size); - npages = howmany(its_tbl_size, PAGE_SIZE); - break; - default: - npages = howmany(page_size, PAGE_SIZE); - break; - } - - /* Allocate required space */ - ptab_vaddr = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, - M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE, 0); - - sc->its_ptabs[tn].ptab_vaddr = ptab_vaddr; - sc->its_ptabs[tn].ptab_pgsz = PAGE_SIZE; - sc->its_ptabs[tn].ptab_npages = npages; - - ptab_paddr = vtophys(ptab_vaddr); - KASSERT((ptab_paddr & GITS_BASER_PA_MASK) == ptab_paddr, - ("%s: Unaligned PA for Interrupt Translation Table", - device_get_name(sc->dev))); - - /* Set defaults: WAWB, IS */ - cache = GITS_BASER_CACHE_WAWB; - share = GITS_BASER_SHARE_IS; - - for (;;) { - nitspages = howmany(its_tbl_size, page_size); - - switch (page_size) { - case PAGE_SIZE: /* 4KB */ - psz = GITS_BASER_PSZ_4K; - break; - case PAGE_SIZE_16K: /* 16KB */ - psz = GITS_BASER_PSZ_4K; - break; - case PAGE_SIZE_64K: /* 64KB */ - psz = GITS_BASER_PSZ_64K; - break; - default: - device_printf(sc->dev, - "Unsupported page size: %zuKB\n", - (page_size / 1024)); - its_free_tables(sc); - return (ENXIO); - } - - /* Clear fields under modification first */ - gits_baser &= ~(GITS_BASER_VALID | - GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | - GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | - GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | - GITS_BASER_SIZE_MASK); - /* Construct register value */ - gits_baser |= - (type << GITS_BASER_TYPE_SHIFT) | - ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | - (cache << GITS_BASER_CACHE_SHIFT) | - (share << GITS_BASER_SHARE_SHIFT) | - (psz << GITS_BASER_PSZ_SHIFT) | - ptab_paddr | (nitspages - 1) | - GITS_BASER_VALID; - - gic_its_write(sc, 8, GITS_BASER(tn), gits_baser); - /* - * Verify. - * Depending on implementation we may encounter - * shareability and page size mismatch. - */ - gits_tmp = gic_its_read(sc, 8, GITS_BASER(tn)); - if (((gits_tmp ^ gits_baser) & GITS_BASER_SHARE_MASK) != 0) { - share = gits_tmp & GITS_BASER_SHARE_MASK; - share >>= GITS_BASER_SHARE_SHIFT; - continue; - } - - if (((gits_tmp ^ gits_baser) & GITS_BASER_PSZ_MASK) != 0) { - switch (page_size) { - case PAGE_SIZE_16K: - /* Drop to 4KB page */ - page_size = PAGE_SIZE; - continue; - case PAGE_SIZE_64K: - /* Drop to 16KB page */ - page_size = PAGE_SIZE_16K; - continue; - } - } - /* - * All possible adjustments should - * be applied by now so just break the loop. - */ - break; - } - /* - * Do not compare Cacheability field since - * it is implementation defined. - */ - gits_tmp &= ~GITS_BASER_CACHE_MASK; - gits_baser &= ~GITS_BASER_CACHE_MASK; - - if (gits_tmp != gits_baser) { - device_printf(sc->dev, - "Could not allocate ITS tables\n"); - its_free_tables(sc); - return (ENXIO); - } - - if (bootverbose) { - if (first) { - device_printf(sc->dev, - "Allocated ITS private tables:\n"); - first = FALSE; - } - device_printf(sc->dev, - "\tPTAB%zu for %s: PA 0x%lx," - " %lu entries," - " cache policy %s, %s shareable," - " page size %zuKB\n", - tn, its_ptab_type[type], ptab_paddr, - (page_size * nitspages) / esize, - its_ptab_cache[cache], its_ptab_share[share], - page_size / 1024); - } - } - - return (0); -} - -static void -its_free_tables(struct gic_v3_its_softc *sc) -{ - vm_offset_t ptab_vaddr; - size_t size; - size_t tn; - - for (tn = 0; tn < GITS_BASER_NUM; tn++) { - ptab_vaddr = sc->its_ptabs[tn].ptab_vaddr; - if (ptab_vaddr == 0) - continue; - size = sc->its_ptabs[tn].ptab_pgsz; - size *= sc->its_ptabs[tn].ptab_npages; - - if ((void *)ptab_vaddr != NULL) - contigfree((void *)ptab_vaddr, size, M_GIC_V3_ITS); - - /* Clear the table description */ - memset(&sc->its_ptabs[tn], 0, sizeof(sc->its_ptabs[tn])); - } -} - -static void -its_init_commandq(struct gic_v3_its_softc *sc) -{ - uint64_t gits_cbaser, gits_tmp; - uint64_t cache, share; - vm_paddr_t cmdq_paddr; - device_t dev; - - dev = sc->dev; - /* Allocate memory for command queue */ - sc->its_cmdq_base = contigmalloc(ITS_CMDQ_SIZE, M_GIC_V3_ITS, - (M_WAITOK | M_ZERO), 0, ~0UL, ITS_CMDQ_SIZE, 0); - /* Set command queue write pointer (command queue empty) */ - sc->its_cmdq_write = sc->its_cmdq_base; - - /* Save command queue pointer and attributes */ - cmdq_paddr = vtophys(sc->its_cmdq_base); - - /* Set defaults: Normal Inner WAWB, IS */ - cache = GITS_CBASER_CACHE_NIWAWB; - share = GITS_CBASER_SHARE_IS; - - gits_cbaser = (cmdq_paddr | - (cache << GITS_CBASER_CACHE_SHIFT) | - (share << GITS_CBASER_SHARE_SHIFT) | - /* Number of 4KB pages - 1 */ - ((ITS_CMDQ_SIZE / PAGE_SIZE) - 1) | - /* Valid bit */ - GITS_CBASER_VALID); - - gic_its_write(sc, 8, GITS_CBASER, gits_cbaser); - gits_tmp = gic_its_read(sc, 8, GITS_CBASER); - - if (((gits_tmp ^ gits_cbaser) & GITS_CBASER_SHARE_MASK) != 0) { - if (bootverbose) { - device_printf(dev, - "Will use cache flushing for commands queue\n"); - } - /* Command queue needs cache flushing */ - sc->its_flags |= ITS_FLAGS_CMDQ_FLUSH; - } - - gic_its_write(sc, 8, GITS_CWRITER, 0x0); -} - -int -its_init_cpu(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - - /* - * NULL in place of the softc pointer means that - * this function was called during GICv3 secondary initialization. - */ - if (sc == NULL) { - if (its_sc != NULL && device_is_attached(its_sc->dev)) { - /* - * XXX ARM64TODO: This is part of the workaround that - * saves ITS software context for further use in - * mask/unmask and here. This should be removed as soon - * as the upper layer is capable of passing the ITS - * context to this function. - */ - sc = its_sc; - } else - return (ENXIO); - - /* Skip if running secondary init on a wrong socket */ - if (sc->its_socket != CPU_CURRENT_SOCKET) - return (ENXIO); - } - - /* - * Check for LPIs support on this Re-Distributor. - */ - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - if ((gic_r_read(gic_sc, 4, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { - if (bootverbose) { - device_printf(sc->dev, - "LPIs not supported on CPU%u\n", PCPU_GET(cpuid)); - } - return (ENXIO); - } - - /* Configure LPIs for this CPU */ - lpi_config_cpu(sc); - - /* Initialize collections */ - its_init_cpu_collection(sc); - - return (0); -} - -static void -its_init_cpu_collection(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - uint64_t typer; - uint64_t target; - vm_offset_t redist_base; - u_int cpuid; - - cpuid = PCPU_GET(cpuid); - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - - typer = gic_its_read(sc, 8, GITS_TYPER); - if ((typer & GITS_TYPER_PTA) != 0) { - redist_base = - rman_get_bushandle(gic_sc->gic_redists.pcpu[cpuid]); - /* - * Target Address correspond to the base physical - * address of Re-Distributors. - */ - target = vtophys(redist_base); - } else { - /* Target Address correspond to unique processor numbers */ - typer = gic_r_read(gic_sc, 8, GICR_TYPER); - target = GICR_TYPER_CPUNUM(typer); - } - - sc->its_cols[cpuid]->col_target = target; - sc->its_cols[cpuid]->col_id = cpuid; - - its_cmd_mapc(sc, sc->its_cols[cpuid], 1); - its_cmd_invall(sc, sc->its_cols[cpuid]); - -} - -static void -lpi_init_conftable(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - vm_offset_t conf_base; - uint8_t prio_default; - - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - /* - * LPI Configuration Table settings. - * Notice that Configuration Table is shared among all - * Re-Distributors, so this is going to be created just once. - */ - conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, - M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); - - if (bootverbose) { - device_printf(sc->dev, - "LPI Configuration Table at PA: 0x%lx\n", - vtophys(conf_base)); - } - - /* - * Let the default priority be aligned with all other - * interrupts assuming that each interrupt is assigned - * MAX priority at startup. MAX priority on the other - * hand cannot be higher than 0xFC for LPIs. - */ - prio_default = GIC_PRIORITY_MAX; - - /* Write each settings byte to LPI configuration table */ - memset((void *)conf_base, - (prio_default & LPI_CONF_PRIO_MASK) | LPI_CONF_GROUP1, - LPI_CONFTAB_SIZE); - - cpu_dcache_wb_range((vm_offset_t)conf_base, roundup2(LPI_CONFTAB_SIZE, - PAGE_SIZE_64K)); - - gic_sc->gic_redists.lpis.conf_base = conf_base; -} - -static void -lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - vm_offset_t pend_base; - u_int cpuid; - - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - - /* - * LPI Pending Table settings. - * This has to be done for each Re-Distributor, hence for each CPU. - */ - for (cpuid = 0; cpuid < mp_ncpus; cpuid++) { - - /* Limit allocation to active CPUs only */ - if (CPU_ISSET(cpuid, &all_cpus) == 0) - continue; - - pend_base = (vm_offset_t)contigmalloc( - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, - (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); - - /* Clean D-cache so that ITS can see zeroed pages */ - cpu_dcache_wb_range((vm_offset_t)pend_base, - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); - - if (bootverbose) { - device_printf(sc->dev, - "LPI Pending Table for CPU%u at PA: 0x%lx\n", - cpuid, vtophys(pend_base)); - } - - gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; - } - - /* Ensure visibility of pend_base addresses on other CPUs */ - wmb(); -} - -static int -lpi_config_cpu(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - vm_offset_t conf_base, pend_base; - uint64_t gicr_xbaser, gicr_temp; - uint64_t cache, share, idbits; - uint32_t gicr_ctlr; - u_int cpuid; - - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - cpuid = PCPU_GET(cpuid); - - /* Ensure data observability on a current CPU */ - rmb(); - - conf_base = gic_sc->gic_redists.lpis.conf_base; - pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; - - /* Disable LPIs */ - gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); - gicr_ctlr &= ~GICR_CTLR_LPI_ENABLE; - gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); - /* Perform full system barrier */ - dsb(sy); - - /* - * Set GICR_PROPBASER - */ - - /* - * Find out how many bits do we need for LPI identifiers. - * Remark 1.: Even though we have (LPI_CONFTAB_SIZE / 8) LPIs - * the notified LPI ID still starts from 8192 - * (GIC_FIRST_LPI). - * Remark 2.: This could be done on compilation time but there - * seems to be no sufficient macro. - */ - idbits = flsl(LPI_CONFTAB_SIZE + GIC_FIRST_LPI) - 1; - - /* Set defaults: Normal Inner WAWB, IS */ - cache = GICR_PROPBASER_CACHE_NIWAWB; - share = GICR_PROPBASER_SHARE_IS; - - gicr_xbaser = vtophys(conf_base) | - ((idbits - 1) & GICR_PROPBASER_IDBITS_MASK) | - (cache << GICR_PROPBASER_CACHE_SHIFT) | - (share << GICR_PROPBASER_SHARE_SHIFT); - - gic_r_write(gic_sc, 8, GICR_PROPBASER, gicr_xbaser); - gicr_temp = gic_r_read(gic_sc, 8, GICR_PROPBASER); - - if (((gicr_xbaser ^ gicr_temp) & GICR_PROPBASER_SHARE_MASK) != 0) { - if (bootverbose) { - device_printf(sc->dev, - "Will use cache flushing for LPI " - "Configuration Table\n"); - } - gic_sc->gic_redists.lpis.flags |= LPI_FLAGS_CONF_FLUSH; - } - - /* - * Set GICR_PENDBASER - */ - - /* Set defaults: Normal Inner WAWB, IS */ - cache = GICR_PENDBASER_CACHE_NIWAWB; - share = GICR_PENDBASER_SHARE_IS; - - gicr_xbaser = vtophys(pend_base) | - (cache << GICR_PENDBASER_CACHE_SHIFT) | - (share << GICR_PENDBASER_SHARE_SHIFT); - - gic_r_write(gic_sc, 8, GICR_PENDBASER, gicr_xbaser); - - /* Enable LPIs */ - gicr_ctlr = gic_r_read(gic_sc, 4, GICR_CTLR); - gicr_ctlr |= GICR_CTLR_LPI_ENABLE; - gic_r_write(gic_sc, 4, GICR_CTLR, gicr_ctlr); - - dsb(sy); - - return (0); -} - -static void -lpi_bitmap_init(struct gic_v3_its_softc *sc) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - uint32_t lpi_id_num; - size_t lpi_chunks_num; - size_t bits_in_chunk; - - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - - lpi_id_num = (1 << gic_sc->gic_idbits) - 1; - /* Substract IDs dedicated for SGIs, PPIs and SPIs */ - lpi_id_num -= GIC_FIRST_LPI; - - sc->its_lpi_maxid = lpi_id_num; - - bits_in_chunk = sizeof(*sc->its_lpi_bitmap) * NBBY; - - /* - * Round up to the number of bits in chunk. - * We will need to take care to avoid using invalid LPI IDs later. - */ - lpi_id_num = roundup2(lpi_id_num, bits_in_chunk); - lpi_chunks_num = lpi_id_num / bits_in_chunk; - - sc->its_lpi_bitmap = - contigmalloc((lpi_chunks_num * sizeof(*sc->its_lpi_bitmap)), - M_GIC_V3_ITS, (M_WAITOK | M_ZERO), 0, ~0UL, - sizeof(*sc->its_lpi_bitmap), 0); -} - -static int -lpi_alloc_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic, - u_int nvecs) -{ - u_int *col_ids; - int fclr; /* First cleared bit */ - bitstr_t *bitmap; - size_t nb, i; - - col_ids = malloc(sizeof(*col_ids) * nvecs, M_GIC_V3_ITS, - (M_NOWAIT | M_ZERO)); - if (col_ids == NULL) - return (ENOMEM); - - mtx_lock_spin(&sc->its_dev_lock); - bitmap = sc->its_lpi_bitmap; - - fclr = 0; -retry: - /* Check other bits - sloooow */ - for (i = 0, nb = fclr; i < nvecs; i++, nb++) { - if (nb > sc->its_lpi_maxid) { - mtx_unlock_spin(&sc->its_dev_lock); - free(col_ids, M_GIC_V3_ITS); - return (EINVAL); - } - - if (isset(bitmap, nb)) { - /* To little free bits in this area. Move on. */ - fclr = nb + 1; - goto retry; - } - } - /* This area is free. Take it. */ - bit_nset(bitmap, fclr, fclr + nvecs - 1); - lpic->lpi_base = fclr + GIC_FIRST_LPI; - lpic->lpi_num = nvecs; - lpic->lpi_busy = 0; - lpic->lpi_free = lpic->lpi_num; - lpic->lpi_col_ids = col_ids; - for (i = 0; i < lpic->lpi_num; i++) { - /* - * Initially all interrupts go to CPU0 but can be moved - * to another CPU by bus_bind_intr() or interrupts shuffling. - */ - lpic->lpi_col_ids[i] = 0; - } - mtx_unlock_spin(&sc->its_dev_lock); - - return (0); -} - -static void -lpi_free_chunk(struct gic_v3_its_softc *sc, struct lpi_chunk *lpic) -{ - int start, end; - - mtx_lock_spin(&sc->its_dev_lock); - KASSERT((lpic->lpi_busy == 0), - ("Trying to free LPI chunk that is still in use.\n")); - /* First bit of this chunk in a global bitmap */ - start = lpic->lpi_base - GIC_FIRST_LPI; - /* and last bit of this chunk... */ - end = start + lpic->lpi_num - 1; - - /* Finally free this chunk */ - bit_nclear(sc->its_lpi_bitmap, start, end); - mtx_unlock_spin(&sc->its_dev_lock); - - free(lpic->lpi_col_ids, M_GIC_V3_ITS); - lpic->lpi_col_ids = NULL; -} - -static void -lpi_configure(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint32_t lpinum, boolean_t unmask) -{ - device_t parent; - struct gic_v3_softc *gic_sc; - uint8_t *conf_byte; - - parent = device_get_parent(sc->dev); - gic_sc = device_get_softc(parent); - - conf_byte = (uint8_t *)gic_sc->gic_redists.lpis.conf_base; - conf_byte += (lpinum - GIC_FIRST_LPI); - - if (unmask) - *conf_byte |= LPI_CONF_ENABLE; - else - *conf_byte &= ~LPI_CONF_ENABLE; - - if ((gic_sc->gic_redists.lpis.flags & LPI_FLAGS_CONF_FLUSH) != 0) { - /* Clean D-cache under configuration byte */ - cpu_dcache_wb_range((vm_offset_t)conf_byte, sizeof(*conf_byte)); - } else { - /* DSB inner shareable, store */ - dsb(ishst); - } - - its_cmd_inv(sc, its_dev, lpinum); -} - -static void -lpi_map_to_device(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint32_t id, uint32_t pid) -{ - - if ((pid < its_dev->lpis.lpi_base) || - (pid >= (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num))) - panic("Trying to map ivalid LPI %u for the device\n", pid); - - its_cmd_mapvi(sc, its_dev, id, pid); -} - -static void -lpi_xmask_irq(device_t parent, uint32_t irq, boolean_t unmask) -{ - struct its_dev *its_dev; - - TAILQ_FOREACH(its_dev, &its_sc->its_dev_list, entry) { - if (irq >= its_dev->lpis.lpi_base && - irq < (its_dev->lpis.lpi_base + its_dev->lpis.lpi_num)) { - lpi_configure(its_sc, its_dev, irq, unmask); - return; - } - } - - panic("Trying to %s not existing LPI: %u\n", - (unmask == TRUE) ? "unmask" : "mask", irq); -} - -int -lpi_migrate(device_t parent, uint32_t irq, u_int cpuid) -{ - struct gic_v3_its_softc *sc; - struct its_dev *its_dev; - struct its_col *col; - - sc = its_sc; - mtx_lock_spin(&sc->its_dev_lock); - its_dev = its_device_find_locked(sc, NULL, irq); - mtx_unlock_spin(&sc->its_dev_lock); - if (its_dev == NULL) { - /* Cannot migrate not configured LPI */ - return (ENXIO); - } - - /* Find local device's interrupt identifier */ - irq = irq - its_dev->lpis.lpi_base; - /* Move interrupt to another collection */ - col = sc->its_cols[cpuid]; - its_cmd_movi(sc, its_dev, col, irq); - its_dev->lpis.lpi_col_ids[irq] = cpuid; - - return (0); -} - -void -lpi_unmask_irq(device_t parent, uint32_t irq) -{ - - lpi_xmask_irq(parent, irq, 1); -} - -void -lpi_mask_irq(device_t parent, uint32_t irq) -{ - - lpi_xmask_irq(parent, irq, 0); -} - -/* - * Commands handling. - */ - -static __inline void -cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) -{ - /* Command field: DW0 [7:0] */ - cmd->cmd_dword[0] &= ~CMD_COMMAND_MASK; - cmd->cmd_dword[0] |= cmd_type; -} - -static __inline void -cmd_format_devid(struct its_cmd *cmd, uint32_t devid) -{ - /* Device ID field: DW0 [63:32] */ - cmd->cmd_dword[0] &= ~CMD_DEVID_MASK; - cmd->cmd_dword[0] |= ((uint64_t)devid << CMD_DEVID_SHIFT); -} - -static __inline void -cmd_format_size(struct its_cmd *cmd, uint16_t size) -{ - /* Size field: DW1 [4:0] */ - cmd->cmd_dword[1] &= ~CMD_SIZE_MASK; - cmd->cmd_dword[1] |= (size & CMD_SIZE_MASK); -} - -static __inline void -cmd_format_id(struct its_cmd *cmd, uint32_t id) -{ - /* ID field: DW1 [31:0] */ - cmd->cmd_dword[1] &= ~CMD_ID_MASK; - cmd->cmd_dword[1] |= id; -} - -static __inline void -cmd_format_pid(struct its_cmd *cmd, uint32_t pid) -{ - /* Physical ID field: DW1 [63:32] */ - cmd->cmd_dword[1] &= ~CMD_PID_MASK; - cmd->cmd_dword[1] |= ((uint64_t)pid << CMD_PID_SHIFT); -} - -static __inline void -cmd_format_col(struct its_cmd *cmd, uint16_t col_id) -{ - /* Collection field: DW2 [16:0] */ - cmd->cmd_dword[2] &= ~CMD_COL_MASK; - cmd->cmd_dword[2] |= col_id; -} - -static __inline void -cmd_format_target(struct its_cmd *cmd, uint64_t target) -{ - /* Target Address field: DW2 [47:16] */ - cmd->cmd_dword[2] &= ~CMD_TARGET_MASK; - cmd->cmd_dword[2] |= (target & CMD_TARGET_MASK); -} - -static __inline void -cmd_format_itt(struct its_cmd *cmd, uint64_t itt) -{ - /* ITT Address field: DW2 [47:8] */ - cmd->cmd_dword[2] &= ~CMD_ITT_MASK; - cmd->cmd_dword[2] |= (itt & CMD_ITT_MASK); -} - -static __inline void -cmd_format_valid(struct its_cmd *cmd, uint8_t valid) -{ - /* Valid field: DW2 [63] */ - cmd->cmd_dword[2] &= ~CMD_VALID_MASK; - cmd->cmd_dword[2] |= ((uint64_t)valid << CMD_VALID_SHIFT); -} - -static __inline void -cmd_fix_endian(struct its_cmd *cmd) -{ - size_t i; - - for (i = 0; i < nitems(cmd->cmd_dword); i++) - cmd->cmd_dword[i] = htole64(cmd->cmd_dword[i]); -} - -static void -its_cmd_movi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - struct its_col *col, uint32_t id) -{ - struct its_cmd_desc desc; - - desc.cmd_type = ITS_CMD_MOVI; - desc.cmd_desc_movi.its_dev = its_dev; - desc.cmd_desc_movi.col = col; - desc.cmd_desc_movi.id = id; - - its_cmd_send(sc, &desc); -} - -static void -its_cmd_mapc(struct gic_v3_its_softc *sc, struct its_col *col, uint8_t valid) -{ - struct its_cmd_desc desc; - - desc.cmd_type = ITS_CMD_MAPC; - desc.cmd_desc_mapc.col = col; - /* - * Valid bit set - map the collection. - * Valid bit cleared - unmap the collection. - */ - desc.cmd_desc_mapc.valid = valid; - - its_cmd_send(sc, &desc); -} - -static void -its_cmd_mapvi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint32_t id, uint32_t pid) -{ - struct its_cmd_desc desc; - struct its_col *col; - u_int col_id; - - col_id = its_dev->lpis.lpi_col_ids[id]; - col = sc->its_cols[col_id]; - - desc.cmd_type = ITS_CMD_MAPVI; - desc.cmd_desc_mapvi.its_dev = its_dev; - desc.cmd_desc_mapvi.col = col; - desc.cmd_desc_mapvi.id = id; - desc.cmd_desc_mapvi.pid = pid; - - its_cmd_send(sc, &desc); -} - -static void __unused -its_cmd_mapi(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) -{ - struct its_cmd_desc desc; - struct its_col *col; - u_int col_id; - uint32_t id; - - KASSERT(pid >= its_dev->lpis.lpi_base, - ("%s: invalid pid: %d for the ITS device", __func__, pid)); - id = pid - its_dev->lpis.lpi_base; - col_id = its_dev->lpis.lpi_col_ids[id]; - col = sc->its_cols[col_id]; - - desc.cmd_type = ITS_CMD_MAPI; - desc.cmd_desc_mapi.its_dev = its_dev; - desc.cmd_desc_mapi.col = col; - desc.cmd_desc_mapi.pid = pid; - - its_cmd_send(sc, &desc); -} - -static void -its_cmd_mapd(struct gic_v3_its_softc *sc, struct its_dev *its_dev, - uint8_t valid) -{ - struct its_cmd_desc desc; - - desc.cmd_type = ITS_CMD_MAPD; - desc.cmd_desc_mapd.its_dev = its_dev; - desc.cmd_desc_mapd.valid = valid; - - its_cmd_send(sc, &desc); -} - -static void -its_cmd_inv(struct gic_v3_its_softc *sc, struct its_dev *its_dev, uint32_t pid) -{ - struct its_cmd_desc desc; - struct its_col *col; - u_int col_id; - uint32_t id; - - KASSERT(pid >= its_dev->lpis.lpi_base, - ("%s: invalid pid: %d for the ITS device", __func__, pid)); - id = pid - its_dev->lpis.lpi_base; - col_id = its_dev->lpis.lpi_col_ids[id]; - col = sc->its_cols[col_id]; - - desc.cmd_type = ITS_CMD_INV; - desc.cmd_desc_inv.pid = pid - its_dev->lpis.lpi_base; - desc.cmd_desc_inv.its_dev = its_dev; - desc.cmd_desc_inv.col = col; - - its_cmd_send(sc, &desc); -} - -static void -its_cmd_invall(struct gic_v3_its_softc *sc, struct its_col *col) -{ - struct its_cmd_desc desc; - - desc.cmd_type = ITS_CMD_INVALL; - desc.cmd_desc_invall.col = col; - - its_cmd_send(sc, &desc); -} - -/* - * Helper routines for commands processing. - */ -static __inline boolean_t -its_cmd_queue_full(struct gic_v3_its_softc *sc) -{ - size_t read_idx, write_idx; - - write_idx = (size_t)(sc->its_cmdq_write - sc->its_cmdq_base); - read_idx = gic_its_read(sc, 4, GITS_CREADR) / sizeof(struct its_cmd); - - /* - * The queue is full when the write offset points - * at the command before the current read offset. - */ - if (((write_idx + 1) % ITS_CMDQ_NENTRIES) == read_idx) - return (TRUE); - - return (FALSE); -} - -static __inline void -its_cmd_sync(struct gic_v3_its_softc *sc, struct its_cmd *cmd) -{ - - if ((sc->its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { - /* Clean D-cache under command. */ - cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); - } else { - /* DSB inner shareable, store */ - dsb(ishst); - } - -} - -static struct its_cmd * -its_cmd_alloc_locked(struct gic_v3_its_softc *sc) -{ - struct its_cmd *cmd; - size_t us_left; - - /* - * XXX ARM64TODO: This is obviously a significant delay. - * The reason for that is that currently the time frames for - * the command to complete (and therefore free the descriptor) - * are not known. - */ - us_left = 1000000; - - mtx_assert(&sc->its_cmd_lock, MA_OWNED); - while (its_cmd_queue_full(sc)) { - if (us_left-- == 0) { - /* Timeout while waiting for free command */ - device_printf(sc->dev, - "Timeout while waiting for free command\n"); - return (NULL); - } - DELAY(1); - } - - cmd = sc->its_cmdq_write; - sc->its_cmdq_write++; - - if (sc->its_cmdq_write == (sc->its_cmdq_base + ITS_CMDQ_NENTRIES)) { - /* Wrap the queue */ - sc->its_cmdq_write = sc->its_cmdq_base; - } - - return (cmd); -} - -static uint64_t -its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) -{ - uint64_t target; - uint8_t cmd_type; - u_int size; - boolean_t error; - - error = FALSE; - cmd_type = desc->cmd_type; - target = ITS_TARGET_NONE; - - switch (cmd_type) { - case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ - target = desc->cmd_desc_movi.col->col_target; - cmd_format_command(cmd, ITS_CMD_MOVI); - cmd_format_id(cmd, desc->cmd_desc_movi.id); - cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); - cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); - break; - case ITS_CMD_SYNC: /* Wait for previous commands completion */ - target = desc->cmd_desc_sync.col->col_target; - cmd_format_command(cmd, ITS_CMD_SYNC); - cmd_format_target(cmd, target); - break; - case ITS_CMD_MAPD: /* Assign ITT to device */ - cmd_format_command(cmd, ITS_CMD_MAPD); - cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); - /* - * Size describes number of bits to encode interrupt IDs - * supported by the device minus one. - * When V (valid) bit is zero, this field should be written - * as zero. - */ - if (desc->cmd_desc_mapd.valid != 0) { - size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); - size = MAX(1, size) - 1; - } else - size = 0; - - cmd_format_size(cmd, size); - cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); - cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); - break; - case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ - target = desc->cmd_desc_mapc.col->col_target; - cmd_format_command(cmd, ITS_CMD_MAPC); - cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); - cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); - cmd_format_target(cmd, target); - break; - case ITS_CMD_MAPVI: - target = desc->cmd_desc_mapvi.col->col_target; - cmd_format_command(cmd, ITS_CMD_MAPVI); - cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); - cmd_format_id(cmd, desc->cmd_desc_mapvi.id); - cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); - cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); - break; - case ITS_CMD_MAPI: - target = desc->cmd_desc_mapi.col->col_target; - cmd_format_command(cmd, ITS_CMD_MAPI); - cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); - cmd_format_id(cmd, desc->cmd_desc_mapi.pid); - cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); - break; - case ITS_CMD_INV: - target = desc->cmd_desc_inv.col->col_target; - cmd_format_command(cmd, ITS_CMD_INV); - cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); - cmd_format_id(cmd, desc->cmd_desc_inv.pid); - break; - case ITS_CMD_INVALL: - cmd_format_command(cmd, ITS_CMD_INVALL); - cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); - break; - default: - error = TRUE; - break; - } - - if (!error) - cmd_fix_endian(cmd); - - return (target); -} - -static __inline uint64_t -its_cmd_cwriter_offset(struct gic_v3_its_softc *sc, struct its_cmd *cmd) -{ - uint64_t off; - - off = (cmd - sc->its_cmdq_base) * sizeof(*cmd); - - return (off); -} - -static void -its_cmd_wait_completion(struct gic_v3_its_softc *sc, struct its_cmd *cmd_first, - struct its_cmd *cmd_last) -{ - uint64_t first, last, read; - size_t us_left; - - /* - * XXX ARM64TODO: This is obviously a significant delay. - * The reason for that is that currently the time frames for - * the command to complete are not known. - */ - us_left = 1000000; - - first = its_cmd_cwriter_offset(sc, cmd_first); - last = its_cmd_cwriter_offset(sc, cmd_last); - - for (;;) { - read = gic_its_read(sc, 8, GITS_CREADR); - if (read < first || read >= last) - break; - - if (us_left-- == 0) { - /* This means timeout */ - device_printf(sc->dev, - "Timeout while waiting for CMD completion.\n"); - return; - } - DELAY(1); - } -} - -static int -its_cmd_send(struct gic_v3_its_softc *sc, struct its_cmd_desc *desc) -{ - struct its_cmd *cmd, *cmd_sync, *cmd_write; - struct its_col col_sync; - struct its_cmd_desc desc_sync; - uint64_t target, cwriter; - - mtx_lock_spin(&sc->its_cmd_lock); - cmd = its_cmd_alloc_locked(sc); - if (cmd == NULL) { - device_printf(sc->dev, "could not allocate ITS command\n"); - mtx_unlock_spin(&sc->its_cmd_lock); - return (EBUSY); - } - - target = its_cmd_prepare(cmd, desc); - its_cmd_sync(sc, cmd); - - if (target != ITS_TARGET_NONE) { - cmd_sync = its_cmd_alloc_locked(sc); - if (cmd_sync == NULL) - goto end; - desc_sync.cmd_type = ITS_CMD_SYNC; - col_sync.col_target = target; - desc_sync.cmd_desc_sync.col = &col_sync; - its_cmd_prepare(cmd_sync, &desc_sync); - its_cmd_sync(sc, cmd_sync); - } -end: - /* Update GITS_CWRITER */ - cwriter = its_cmd_cwriter_offset(sc, sc->its_cmdq_write); - gic_its_write(sc, 8, GITS_CWRITER, cwriter); - cmd_write = sc->its_cmdq_write; - mtx_unlock_spin(&sc->its_cmd_lock); - - its_cmd_wait_completion(sc, cmd, cmd_write); - - return (0); -} - -/* Find ITS device descriptor by pci_dev or irq number */ -static struct its_dev * -its_device_find_locked(struct gic_v3_its_softc *sc, device_t pci_dev, - uint32_t irq) -{ - struct its_dev *its_dev; - struct lpi_chunk *lpis; - - mtx_assert(&sc->its_dev_lock, MA_OWNED); - KASSERT((pci_dev == NULL || irq == 0), - ("%s: Can't search by both pci_dev and irq number", __func__)); - /* Find existing device if any */ - TAILQ_FOREACH(its_dev, &sc->its_dev_list, entry) { - if (pci_dev != NULL) { - if (its_dev->pci_dev == pci_dev) - return (its_dev); - } else if (irq != 0) { - lpis = &its_dev->lpis; - if ((irq >= lpis->lpi_base) && - (irq < (lpis->lpi_base + lpis->lpi_num))) - return (its_dev); - } - } - - return (NULL); -} - -static struct its_dev * -its_device_alloc(struct gic_v3_its_softc *sc, device_t pci_dev, - u_int nvecs) -{ - struct its_dev *newdev; - vm_offset_t itt_addr; - uint64_t typer; - uint32_t devid; - size_t esize; - int err; - - mtx_lock_spin(&sc->its_dev_lock); - /* Find existing device if any */ - newdev = its_device_find_locked(sc, pci_dev, 0); - mtx_unlock_spin(&sc->its_dev_lock); - if (newdev != NULL) - return (newdev); - - devid = its_get_devid(pci_dev); - - /* There was no previously created device. Create one now */ - newdev = malloc(sizeof(*newdev), M_GIC_V3_ITS, (M_NOWAIT | M_ZERO)); - if (newdev == NULL) - return (NULL); - - newdev->pci_dev = pci_dev; - newdev->devid = devid; - - err = lpi_alloc_chunk(sc, &newdev->lpis, nvecs); - if (err != 0) { - free(newdev, M_GIC_V3_ITS); - return (NULL); - } - - /* Get ITT entry size */ - typer = gic_its_read(sc, 8, GITS_TYPER); - esize = GITS_TYPER_ITTES(typer); - /* - * Allocate ITT for this device. - * PA has to be 256 B aligned. At least two entries for device. - */ - newdev->itt_size = roundup2(roundup2(nvecs, 2) * esize, 0x100); - itt_addr = (vm_offset_t)contigmalloc( - newdev->itt_size, M_GIC_V3_ITS, (M_NOWAIT | M_ZERO), - 0, ~0UL, 0x100, 0); - if (itt_addr == 0) { - lpi_free_chunk(sc, &newdev->lpis); - free(newdev, M_GIC_V3_ITS); - return (NULL); - } - - mtx_lock_spin(&sc->its_dev_lock); - newdev->itt = itt_addr; - TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); - mtx_unlock_spin(&sc->its_dev_lock); - - /* Map device to its ITT */ - its_cmd_mapd(sc, newdev, 1); - - return (newdev); -} - -static void -its_device_free(struct gic_v3_its_softc *sc, device_t pci_dev, - u_int nvecs) -{ - struct its_dev *odev; - - mtx_lock_spin(&sc->its_dev_lock); - /* Find existing device if any */ - odev = its_device_find_locked(sc, pci_dev, 0); - if (odev == NULL) { - mtx_unlock_spin(&sc->its_dev_lock); - return; - } - - KASSERT((nvecs <= odev->lpis.lpi_num) && (nvecs <= odev->lpis.lpi_busy), - ("Invalid number of LPI vectors to free %d (total %d) (busy %d)", - nvecs, odev->lpis.lpi_num, odev->lpis.lpi_busy)); - /* Just decrement number of busy LPIs in chunk */ - odev->lpis.lpi_busy -= nvecs; - if (odev->lpis.lpi_busy != 0) { - mtx_unlock_spin(&sc->its_dev_lock); - return; - } - - /* - * At that point we know that there are no busy LPIs for this device. - * Entire ITS device can now be removed. - */ - mtx_unlock_spin(&sc->its_dev_lock); - /* Unmap device in ITS */ - its_cmd_mapd(sc, odev, 0); - /* Free ITT */ - KASSERT(odev->itt != 0, ("Invalid ITT in valid ITS device")); - contigfree((void *)odev->itt, odev->itt_size, M_GIC_V3_ITS); - /* Free chunk */ - lpi_free_chunk(sc, &odev->lpis); - /* Free device */ - mtx_lock_spin(&sc->its_dev_lock); - TAILQ_REMOVE(&sc->its_dev_list, odev, entry); - mtx_unlock_spin(&sc->its_dev_lock); - free((void *)odev, M_GIC_V3_ITS); - -} - -static __inline void -its_device_asign_lpi_locked(struct gic_v3_its_softc *sc, - struct its_dev *its_dev, u_int *irq) -{ - - mtx_assert(&sc->its_dev_lock, MA_OWNED); - if (its_dev->lpis.lpi_free == 0) { - panic("Requesting more LPIs than allocated for this device. " - "LPI num: %u, free %u", its_dev->lpis.lpi_num, - its_dev->lpis.lpi_free); - } - *irq = its_dev->lpis.lpi_base + (its_dev->lpis.lpi_num - - its_dev->lpis.lpi_free); - its_dev->lpis.lpi_free--; - its_dev->lpis.lpi_busy++; -} - -/* - * ITS quirks. - * Add vendor specific PCI devid function here. - */ -static uint32_t -its_get_devbits_thunder(device_t dev) -{ - uint32_t devid_bits; - - /* - * GITS_TYPER[17:13] of ThunderX reports that device IDs - * are to be 21 bits in length. - * The entry size of the ITS table can be read from GITS_BASERn[52:48] - * and on ThunderX is supposed to be 8 bytes in length (for device - * table). Finally the page size that is to be used by ITS to access - * this table will be set to 64KB. - * - * This gives 0x200000 entries of size 0x8 bytes covered by 256 pages - * each of which 64KB in size. The number of pages (minus 1) should - * then be written to GITS_BASERn[7:0]. In that case this value would - * be 0xFF but on ThunderX the maximum value that HW accepts is 0xFD. - * - * Set arbitrary number of device ID bits to 20 in order to limit - * the number of entries in ITS device table to 0x100000 and hence - * the table size to 8MB. - */ - devid_bits = 20; - if (bootverbose) { - device_printf(dev, - "Limiting number of Device ID bits implemented to %d\n", - devid_bits); - } - - return (devid_bits); -} - -static __inline uint32_t -its_get_devbits_default(device_t dev) -{ - uint64_t gits_typer; - struct gic_v3_its_softc *sc; - - sc = device_get_softc(dev); - - gits_typer = gic_its_read(sc, 8, GITS_TYPER); - - return (GITS_TYPER_DEVB(gits_typer)); -} - -static uint32_t -its_get_devbits(device_t dev) -{ - const struct its_quirks *quirk; - size_t i; - - for (i = 0; i < nitems(its_quirks); i++) { - quirk = &its_quirks[i]; - if (CPU_MATCH_RAW(quirk->cpuid_mask, quirk->cpuid)) { - if (quirk->devbits_func != NULL) - return ((*quirk->devbits_func)(dev)); - } - } - - return (its_get_devbits_default(dev)); -} - -static uint32_t -its_get_devid(device_t pci_dev) -{ - uintptr_t id; - - if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) - panic("its_get_devid: Unable to get the MSI DeviceID"); - - return (id); -} - -/* - * Message signalled interrupts handling. - */ - -/* - * XXX ARM64TODO: Watch out for "irq" type. - * - * In theory GIC can handle up to (2^32 - 1) interrupt IDs whereas - * we pass "irq" pointer of type integer. This is obviously wrong but - * is determined by the way as PCI layer wants it to be done. - */ -int -gic_v3_its_alloc_msix(device_t dev, device_t pci_dev, int *irq) -{ - struct gic_v3_its_softc *sc; - struct its_dev *its_dev; - u_int nvecs; - - sc = device_get_softc(dev); - - nvecs = PCI_MSIX_NUM(pci_dev); - - /* - * Allocate device as seen by ITS if not already available. - * Notice that MSI-X interrupts are allocated on one-by-one basis. - */ - its_dev = its_device_alloc(sc, pci_dev, nvecs); - if (its_dev == NULL) - return (ENOMEM); - - mtx_lock_spin(&sc->its_dev_lock); - its_device_asign_lpi_locked(sc, its_dev, irq); - mtx_unlock_spin(&sc->its_dev_lock); - - return (0); -} - -int -gic_v3_its_release_msix(device_t dev, device_t pci_dev, int irq __unused) -{ - - struct gic_v3_its_softc *sc; - - sc = device_get_softc(dev); - its_device_free(sc, pci_dev, 1); - - return (0); -} - -int -gic_v3_its_alloc_msi(device_t dev, device_t pci_dev, int count, int *irqs) -{ - struct gic_v3_its_softc *sc; - struct its_dev *its_dev; - - sc = device_get_softc(dev); - - /* Allocate device as seen by ITS if not already available. */ - its_dev = its_device_alloc(sc, pci_dev, count); - if (its_dev == NULL) - return (ENOMEM); - - mtx_lock_spin(&sc->its_dev_lock); - for (; count > 0; count--) { - its_device_asign_lpi_locked(sc, its_dev, irqs); - irqs++; - } - mtx_unlock_spin(&sc->its_dev_lock); - - return (0); -} - -int -gic_v3_its_release_msi(device_t dev, device_t pci_dev, int count, - int *irqs __unused) -{ - struct gic_v3_its_softc *sc; - - sc = device_get_softc(dev); - its_device_free(sc, pci_dev, count); - - return (0); -} - -int -gic_v3_its_map_msi(device_t dev, device_t pci_dev, int irq, uint64_t *addr, - uint32_t *data) -{ - struct gic_v3_its_softc *sc; - bus_space_handle_t its_bsh; - struct its_dev *its_dev; - uint64_t its_pa; - uint32_t id; - - sc = device_get_softc(dev); - /* Verify that this device is allocated and owns this LPI */ - mtx_lock_spin(&sc->its_dev_lock); - its_dev = its_device_find_locked(sc, pci_dev, 0); - mtx_unlock_spin(&sc->its_dev_lock); - if (its_dev == NULL) - return (EINVAL); - - id = irq - its_dev->lpis.lpi_base; - lpi_map_to_device(sc, its_dev, id, irq); - - its_bsh = rman_get_bushandle(&sc->its_res[0]); - its_pa = vtophys(its_bsh); - - *addr = (its_pa + GITS_TRANSLATER); - *data = id; - - return (0); -} diff --git a/sys/arm64/arm64/intr_machdep.c b/sys/arm64/arm64/intr_machdep.c deleted file mode 100644 index d99303e69570..000000000000 --- a/sys/arm64/arm64/intr_machdep.c +++ /dev/null @@ -1,584 +0,0 @@ -/*- - * Copyright (c) 1991 The Regents of the University of California. - * Copyright (c) 2002 Benno Rice. - * Copyright (c) 2014 The FreeBSD Foundation - * All rights reserved. - * - * This software was developed by Semihalf under - * the sponsorship of the FreeBSD Foundation. - * - * This code is derived from software contributed by - * William Jolitz (Berkeley) and Benno Rice. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * form: src/sys/powerpc/powerpc/intr_machdep.c, r271712 2014/09/17 - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#ifdef SMP -#include -#endif - -#include "pic_if.h" - -#define MAX_STRAY_LOG 5 -#define INTRNAME_LEN (MAXCOMLEN + 1) - -#define NIRQS 1024 /* Maximum number of interrupts in the system */ - -static MALLOC_DEFINE(M_INTR, "intr", "Interrupt Services"); - -/* - * Linked list of interrupts that have been set-up. - * Each element holds the interrupt description - * and has to be allocated and freed dynamically. - */ -static SLIST_HEAD(, arm64_intr_entry) irq_slist_head = - SLIST_HEAD_INITIALIZER(irq_slist_head); - -struct arm64_intr_entry { - SLIST_ENTRY(arm64_intr_entry) entries; - struct intr_event *i_event; - - enum intr_trigger i_trig; - enum intr_polarity i_pol; - - u_int i_hw_irq; /* Physical interrupt number */ - u_int i_cntidx; /* Index in intrcnt table */ - u_int i_handlers; /* Allocated handlers */ - u_int i_cpu; /* Assigned CPU */ - u_long *i_cntp; /* Interrupt hit counter */ -}; - -/* Counts and names for statistics - see sys/sys/interrupt.h */ -/* Tables are indexed by i_cntidx */ -u_long intrcnt[NIRQS]; -char intrnames[NIRQS * INTRNAME_LEN]; -size_t sintrcnt = sizeof(intrcnt); -size_t sintrnames = sizeof(intrnames); - -static u_int intrcntidx; /* Current index into intrcnt table */ -static u_int arm64_nintrs; /* Max interrupts number of the root PIC */ -static u_int arm64_nstray; /* Number of received stray interrupts */ -static device_t root_pic; /* PIC device for all incoming interrupts */ -static device_t msi_pic; /* Device which handles MSI/MSI-X interrupts */ -static struct mtx intr_list_lock; - -static void -intr_init(void *dummy __unused) -{ - - mtx_init(&intr_list_lock, "intr sources lock", NULL, MTX_SPIN); -} -SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); - -/* - * Helper routines. - */ - -/* Set interrupt name for statistics */ -static void -intrcnt_setname(const char *name, u_int idx) -{ - - snprintf(&intrnames[idx * INTRNAME_LEN], INTRNAME_LEN, "%-*s", - INTRNAME_LEN - 1, name); -} - -/* - * Find the interrupt descriptor in the list - * based on the hardware IRQ number. - */ -static __inline struct arm64_intr_entry * -intr_lookup_locked(u_int hw_irq) -{ - struct arm64_intr_entry *intr; - - mtx_assert(&intr_list_lock, MA_OWNED); - SLIST_FOREACH(intr, &irq_slist_head, entries) { - if (intr->i_hw_irq == hw_irq) - return (intr); - } - return (NULL); -} -/* - * Get intr structure for the given interrupt number. - * Allocate one if this is the first time. - */ -static struct arm64_intr_entry * -intr_allocate(u_int hw_irq) -{ - struct arm64_intr_entry *intr; - - /* Check if already allocated */ - mtx_lock_spin(&intr_list_lock); - intr = intr_lookup_locked(hw_irq); - mtx_unlock_spin(&intr_list_lock); - if (intr != NULL) - return (intr); - - /* Do not alloc another intr when max number of IRQs has been reached */ - if (intrcntidx >= NIRQS) - return (NULL); - - intr = malloc(sizeof(*intr), M_INTR, M_NOWAIT); - if (intr == NULL) - return (NULL); - - /* The default CPU is 0 but can be changed later by bind or shuffle */ - intr->i_cpu = 0; - intr->i_event = NULL; - intr->i_handlers = 0; - intr->i_trig = INTR_TRIGGER_CONFORM; - intr->i_pol = INTR_POLARITY_CONFORM; - intr->i_cntidx = atomic_fetchadd_int(&intrcntidx, 1); - intr->i_cntp = &intrcnt[intr->i_cntidx]; - intr->i_hw_irq = hw_irq; - mtx_lock_spin(&intr_list_lock); - SLIST_INSERT_HEAD(&irq_slist_head, intr, entries); - mtx_unlock_spin(&intr_list_lock); - - return intr; -} - -static int -intr_assign_cpu(void *arg, int cpu) -{ -#ifdef SMP - struct arm64_intr_entry *intr; - int error; - - if (root_pic == NULL) - panic("Cannot assing interrupt to CPU. No PIC configured"); - /* - * Set the interrupt to CPU affinity. - * Do not configure this in hardware during early boot. - * We will pick up the assignment once the APs are started. - */ - if (cpu != NOCPU) { - intr = arg; - if (!cold && smp_started) { - /* - * Bind the interrupt immediately - * if SMP is up and running. - */ - error = PIC_BIND(root_pic, intr->i_hw_irq, cpu); - if (error == 0) - intr->i_cpu = cpu; - } else { - /* Postpone binding until SMP is operational */ - intr->i_cpu = cpu; - error = 0; - } - } else - error = 0; - - return (error); -#else - return (EOPNOTSUPP); -#endif -} - -static void -intr_pre_ithread(void *arg) -{ - struct arm64_intr_entry *intr = arg; - - PIC_PRE_ITHREAD(root_pic, intr->i_hw_irq); -} - -static void -intr_post_ithread(void *arg) -{ - struct arm64_intr_entry *intr = arg; - - PIC_POST_ITHREAD(root_pic, intr->i_hw_irq); -} - -static void -intr_post_filter(void *arg) -{ - struct arm64_intr_entry *intr = arg; - - PIC_POST_FILTER(root_pic, intr->i_hw_irq); -} - -/* - * Register PIC driver. - * This is intended to be called by the very first PIC driver - * at the end of the successful attach. - * Note that during boot this can be called after first references - * to bus_setup_intr() so it is required to not use root_pic if it - * is not 100% safe. - */ -void -arm_register_root_pic(device_t dev, u_int nirq) -{ - - KASSERT(root_pic == NULL, ("Unable to set the pic twice")); - KASSERT(nirq <= NIRQS, ("PIC is trying to handle too many IRQs")); - - arm64_nintrs = NIRQS; /* Number of IRQs limited only by array size */ - root_pic = dev; -} - -/* Register device which allocates MSI interrupts */ -void -arm_register_msi_pic(device_t dev) -{ - - KASSERT(msi_pic == NULL, ("Unable to set msi_pic twice")); - msi_pic = dev; -} - -int -arm_alloc_msi(device_t pci, device_t child, int count, int maxcount, int *irqs) -{ - - return (PIC_ALLOC_MSI(msi_pic, child, count, irqs)); -} - -int -arm_release_msi(device_t pci, device_t child, int count, int *irqs) -{ - - return (PIC_RELEASE_MSI(msi_pic, child, count, irqs)); -} - -int -arm_map_msi(device_t pci, device_t child, int irq, uint64_t *addr, uint32_t *data) -{ - - return (PIC_MAP_MSI(msi_pic, child, irq, addr, data)); -} - -int -arm_alloc_msix(device_t pci, device_t child, int *irq) -{ - - return (PIC_ALLOC_MSIX(msi_pic, child, irq)); -} - -int -arm_release_msix(device_t pci, device_t child, int irq) -{ - - return (PIC_RELEASE_MSIX(msi_pic, child, irq)); -} - - -/* - * Finalize interrupts bring-up (should be called from configure_final()). - * Enables all interrupts registered by bus_setup_intr() during boot - * as well as unlocks interrups reception on primary CPU. - */ -int -arm_enable_intr(void) -{ - struct arm64_intr_entry *intr; - - if (root_pic == NULL) - panic("Cannot enable interrupts. No PIC configured"); - - /* - * Iterate through all possible interrupts and perform - * configuration if the interrupt is registered. - */ - SLIST_FOREACH(intr, &irq_slist_head, entries) { - /* - * XXX: In case we allowed to set up interrupt whose number - * exceeds maximum number of interrupts for the root PIC - * disable it and print proper error message. - * - * This can happen only when calling bus_setup_intr() - * before the interrupt controller is attached. - */ - if (intr->i_cntidx >= arm64_nintrs) { - /* Better fail when IVARIANTS enabled */ - KASSERT(0, ("%s: Interrupt %u cannot be handled by the " - "registered PIC. Max interrupt number: %u", __func__, - intr->i_cntidx, arm64_nintrs - 1)); - /* Print message and disable otherwise */ - printf("ERROR: Cannot enable irq %u. Disabling.\n", - intr->i_cntidx); - PIC_MASK(root_pic, intr->i_hw_irq); - } - - if (intr->i_trig != INTR_TRIGGER_CONFORM || - intr->i_pol != INTR_POLARITY_CONFORM) { - PIC_CONFIG(root_pic, intr->i_hw_irq, - intr->i_trig, intr->i_pol); - } - - if (intr->i_handlers > 0) - PIC_UNMASK(root_pic, intr->i_hw_irq); - - } - /* Enable interrupt reception on this CPU */ - intr_enable(); - - return (0); -} - -int -arm_setup_intr(const char *name, driver_filter_t *filt, driver_intr_t handler, - void *arg, u_int hw_irq, enum intr_type flags, void **cookiep) -{ - struct arm64_intr_entry *intr; - int error; - - intr = intr_allocate(hw_irq); - if (intr == NULL) - return (ENOMEM); - - /* - * Watch out for interrupts' numbers. - * If this is a system boot then don't allow to overfill interrupts - * table (the interrupts will be deconfigured in arm_enable_intr()). - */ - if (intr->i_cntidx >= NIRQS) - return (EINVAL); - - if (intr->i_event == NULL) { - error = intr_event_create(&intr->i_event, (void *)intr, 0, - hw_irq, intr_pre_ithread, intr_post_ithread, - intr_post_filter, intr_assign_cpu, "irq%u", hw_irq); - if (error) - return (error); - } - - error = intr_event_add_handler(intr->i_event, name, filt, handler, arg, - intr_priority(flags), flags, cookiep); - - if (!error) { - intrcnt_setname(intr->i_event->ie_fullname, intr->i_cntidx); - intr->i_handlers++; - - if (!cold && intr->i_handlers == 1) { - if (intr->i_trig != INTR_TRIGGER_CONFORM || - intr->i_pol != INTR_POLARITY_CONFORM) { - PIC_CONFIG(root_pic, intr->i_hw_irq, intr->i_trig, - intr->i_pol); - } - - PIC_UNMASK(root_pic, intr->i_hw_irq); - } - } - - return (error); -} - -int -intr_irq_remove_handler(device_t dev, u_int irq, void *cookie) -{ - struct arm64_intr_entry *intr; - int error; - - intr = intr_handler_source(cookie); - error = intr_event_remove_handler(cookie); - if (!error) { - intr->i_handlers--; - if (intr->i_handlers == 0) - PIC_MASK(root_pic, intr->i_hw_irq); - intrcnt_setname(intr->i_event->ie_fullname, intr->i_cntidx); - } - - return (error); -} - -int -intr_irq_config(u_int hw_irq, enum intr_trigger trig, enum intr_polarity pol) -{ - struct arm64_intr_entry *intr; - - mtx_lock_spin(&intr_list_lock); - intr = intr_lookup_locked(hw_irq); - mtx_unlock_spin(&intr_list_lock); - if (intr == NULL) - return (EINVAL); - - intr->i_trig = trig; - intr->i_pol = pol; - - if (!cold && root_pic != NULL) - PIC_CONFIG(root_pic, intr->i_hw_irq, trig, pol); - - return (0); -} - -void -arm_dispatch_intr(u_int hw_irq, struct trapframe *tf) -{ - struct arm64_intr_entry *intr; - - mtx_lock_spin(&intr_list_lock); - intr = intr_lookup_locked(hw_irq); - mtx_unlock_spin(&intr_list_lock); - if (intr == NULL) - goto stray; - - (*intr->i_cntp)++; - - if (!intr_event_handle(intr->i_event, tf)) - return; - -stray: - if (arm64_nstray < MAX_STRAY_LOG) { - arm64_nstray++; - printf("Stray IRQ %u\n", hw_irq); - if (arm64_nstray >= MAX_STRAY_LOG) { - printf("Got %d stray IRQs. Not logging anymore.\n", - MAX_STRAY_LOG); - } - } - - if (intr != NULL) - PIC_MASK(root_pic, intr->i_hw_irq); -} - -void -intr_irq_handler(struct trapframe *tf) -{ - - critical_enter(); - PIC_DISPATCH(root_pic, tf); - critical_exit(); -#ifdef HWPMC_HOOKS - if (pmc_hook && (PCPU_GET(curthread)->td_pflags & TDP_CALLCHAIN)) - pmc_hook(PCPU_GET(curthread), PMC_FN_USER_CALLCHAIN, tf); -#endif -} - -#ifdef SMP -static void -arm_intr_smp_init(void *dummy __unused) -{ - struct arm64_intr_entry *intr; - int error; - - if (root_pic == NULL) - panic("Cannot assing interrupts to CPUs. No PIC configured"); - - mtx_lock_spin(&intr_list_lock); - SLIST_FOREACH(intr, &irq_slist_head, entries) { - mtx_unlock_spin(&intr_list_lock); - error = PIC_BIND(root_pic, intr->i_hw_irq, intr->i_cpu); - if (error != 0) - intr->i_cpu = 0; - mtx_lock_spin(&intr_list_lock); - } - mtx_unlock_spin(&intr_list_lock); -} -SYSINIT(arm_intr_smp_init, SI_SUB_SMP, SI_ORDER_ANY, arm_intr_smp_init, NULL); - -/* Attempt to bind the specified IRQ to the specified CPU. */ -int -intr_irq_bind(u_int hw_irq, int cpu) -{ - struct arm64_intr_entry *intr; - - mtx_lock_spin(&intr_list_lock); - intr = intr_lookup_locked(hw_irq); - mtx_unlock_spin(&intr_list_lock); - if (intr == NULL) - return (EINVAL); - - return (intr_event_bind(intr->i_event, cpu)); -} - -void -arm_setup_ipihandler(driver_filter_t *filt, u_int ipi) -{ - - arm_setup_intr("ipi", filt, NULL, (void *)((uintptr_t)ipi | 1<<16), ipi, - INTR_TYPE_MISC | INTR_EXCL, NULL); - arm_unmask_ipi(ipi); -} - -void -arm_unmask_ipi(u_int ipi) -{ - - PIC_UNMASK(root_pic, ipi); -} - -void -arm_init_secondary(void) -{ - - PIC_INIT_SECONDARY(root_pic); -} - -/* Sending IPI */ -void -ipi_all_but_self(u_int ipi) -{ - cpuset_t other_cpus; - - other_cpus = all_cpus; - CPU_CLR(PCPU_GET(cpuid), &other_cpus); - - CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); - PIC_IPI_SEND(root_pic, other_cpus, ipi); -} - -void -ipi_cpu(int cpu, u_int ipi) -{ - cpuset_t cpus; - - CPU_ZERO(&cpus); - CPU_SET(cpu, &cpus); - - CTR2(KTR_SMP, "ipi_cpu: cpu: %d, ipi: %x", cpu, ipi); - PIC_IPI_SEND(root_pic, cpus, ipi); -} - -void -ipi_selected(cpuset_t cpus, u_int ipi) -{ - - CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); - PIC_IPI_SEND(root_pic, cpus, ipi); -} - -#endif diff --git a/sys/arm64/arm64/pic_if.m b/sys/arm64/arm64/pic_if.m deleted file mode 100644 index 33d1bcd00aa5..000000000000 --- a/sys/arm64/arm64/pic_if.m +++ /dev/null @@ -1,176 +0,0 @@ -#- -# Copyright (c) 1998 Doug Rabson -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# 1. Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# 2. Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -# SUCH DAMAGE. -# -# from: src/sys/kern/bus_if.m,v 1.21 2002/04/21 11:16:10 markm Exp -# $FreeBSD$ -# - -#include -#include -#include - -INTERFACE pic; - -CODE { - static int pic_bind_default(device_t dev, u_int irq, u_int cpu) - { - - return (EOPNOTSUPP); - } - - static void pic_translate_code_default(device_t dev, u_int irq, - int code, enum intr_trigger *trig, enum intr_polarity *pol) - { - *trig = INTR_TRIGGER_CONFORM; - *pol = INTR_POLARITY_CONFORM; - } - - static void pic_pre_ithread(device_t dev, u_int irq) - { - PIC_MASK(dev, irq); - PIC_EOI(dev, irq); - } - - static void pic_post_ithread(device_t dev, u_int irq) - { - PIC_UNMASK(dev, irq); - } - - static void pic_post_filter(device_t dev, u_int irq) - { - PIC_EOI(dev, irq); - } -}; - -METHOD int bind { - device_t dev; - u_int irq; - u_int cpu; -} DEFAULT pic_bind_default; - -METHOD void translate_code { - device_t dev; - u_int irq; - int code; - enum intr_trigger *trig; - enum intr_polarity *pol; -} DEFAULT pic_translate_code_default; - -METHOD void config { - device_t dev; - u_int irq; - enum intr_trigger trig; - enum intr_polarity pol; -}; - -METHOD void dispatch { - device_t dev; - struct trapframe *tf; -}; - -METHOD void enable { - device_t dev; - u_int irq; - u_int vector; -}; - -METHOD void pre_ithread { - device_t dev; - u_int irq; -} DEFAULT pic_pre_ithread; - -METHOD void post_ithread { - device_t dev; - u_int irq; -} DEFAULT pic_post_ithread; - -METHOD void post_filter { - device_t dev; - u_int irq; -} DEFAULT pic_post_filter; - -METHOD void eoi { - device_t dev; - u_int irq; -}; - -METHOD void ipi { - device_t dev; - u_int cpu; -}; - -METHOD void mask { - device_t dev; - u_int irq; -}; - -METHOD void unmask { - device_t dev; - u_int irq; -}; - -METHOD void init_secondary { - device_t dev; -}; - -METHOD void ipi_send { - device_t dev; - cpuset_t cpus; - u_int ipi; -}; - -METHOD int alloc_msi { - device_t dev; - device_t pci_dev; - int count; - int *irqs; -}; - -METHOD int alloc_msix { - device_t dev; - device_t pci_dev; - int *irq; -}; - -METHOD int map_msi { - device_t dev; - device_t pci_dev; - int irq; - uint64_t *addr; - uint32_t *data; -}; - -METHOD int release_msi { - device_t dev; - device_t pci_dev; - int count; - int *irqs; -}; - -METHOD int release_msix { - device_t dev; - device_t pci_dev; - int irq; -}; diff --git a/sys/conf/files.arm64 b/sys/conf/files.arm64 index 3f0d62728b3a..6a78f0821f48 100644 --- a/sys/conf/files.arm64 +++ b/sys/conf/files.arm64 @@ -25,15 +25,10 @@ arm64/arm64/disassem.c optional ddb arm64/arm64/dump_machdep.c standard arm64/arm64/elf_machdep.c standard arm64/arm64/exception.S standard -arm64/arm64/gic.c optional !intrng arm64/arm64/gicv3_its.c optional intrng -arm64/arm64/gic_acpi.c optional !intrng acpi -arm64/arm64/gic_fdt.c optional !intrng fdt arm64/arm64/gic_v3.c standard arm64/arm64/gic_v3_fdt.c optional fdt -arm64/arm64/gic_v3_its.c optional !intrng arm64/arm64/identcpu.c standard -arm64/arm64/intr_machdep.c optional !intrng arm64/arm64/in_cksum.c optional inet | inet6 arm64/arm64/locore.S standard no-obj arm64/arm64/machdep.c standard @@ -42,7 +37,6 @@ arm64/arm64/minidump_machdep.c standard arm64/arm64/mp_machdep.c optional smp arm64/arm64/nexus.c standard arm64/arm64/ofw_machdep.c optional fdt -arm64/arm64/pic_if.m optional !intrng arm64/arm64/pmap.c standard arm64/arm64/stack_machdep.c optional ddb | stack arm64/arm64/support.S standard