d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

MFC: Revamp the MSI/MSI-X code a bit to achieve two main goals:

Browse Source 
- Simplify the amount of work that has be done for each architecture by
  pushing more of the truly MI code down into the PCI bus driver.
- Don't bind MSI-X indicies to IRQs so that we can allow a driver to map
  multiple MSI-X messages into a single IRQ when handling a message
  shortage.

Note that as with the previous MSI MFC, this does not yet include the
'pci_remap_msix()' function.
This commit is contained in:
jhb 2007-08-15 20:56:10 +00:00
parent f0f120d153
commit 24a23340cc
18 changed files with 590 additions and 337 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
sys/amd64/amd64/mptable_pci.c
View File

@ -72,7 +72,7 @@ mptable_hostb_attach(device_t dev)
return (bus_generic_attach(dev));
}
/* Pass MSI alloc requests up to the nexus. */
/* Pass MSI requests up to the nexus. */
static int
mptable_hostb_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
int *irqs)
@ -85,12 +85,22 @@ mptable_hostb_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
}
static int
mptable_hostb_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
mptable_hostb_alloc_msix(device_t pcib, device_t dev, int *irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
static int
mptable_hostb_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
static device_method_t mptable_hostb_methods[] = {
@ -120,8 +130,8 @@ static device_method_t mptable_hostb_methods[] = {
DEVMETHOD(pcib_alloc_msi, mptable_hostb_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, mptable_hostb_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, mptable_hostb_map_msi),
{ 0, 0 }
};
@ -177,8 +187,8 @@ static device_method_t mptable_pcib_pci_methods[] = {
DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, pcib_map_msi),
{0, 0}
};

168
sys/amd64/amd64/msi.c
View File

@ -99,21 +99,20 @@ static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
* assigned an ID by the system; however, a group will use the ID from
* the first message.
*
* For MSI-X, each message is isolated, and msi_index indicates the
* index of this message in the device's MSI-X table.
* For MSI-X, each message is isolated.
*/
struct msi_intsrc {
struct intsrc msi_intsrc;
device_t msi_dev; /* Owning device. (g) */
struct msi_intsrc *msi_first; /* First source in group. */
u_int msi_irq; /* IRQ cookie. */
u_int msi_index; /* Index of this message. */
u_int msi_msix; /* MSI-X message. */
u_int msi_vector:8; /* IDT vector. */
u_int msi_cpu:8; /* Local APIC ID. (g) */
u_int msi_count:8; /* Messages in this group. (g) */
};
static struct msi_intsrc *msi_create_source(u_int irq);
static void msi_enable_source(struct intsrc *isrc);
static void msi_disable_source(struct intsrc *isrc, int eoi);
static void msi_eoi_source(struct intsrc *isrc);
@ -123,16 +122,10 @@ static int msi_source_pending(struct intsrc *isrc);
static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
enum intr_polarity pol);
static void msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
static void msix_enable_intr(struct intsrc *isrc);
static int msix_source_pending(struct intsrc *isrc);
static void msix_assign_cpu(struct intsrc *isrc, u_int apic_id);
struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
msi_enable_intr, msi_vector, msi_source_pending,
NULL, NULL, msi_config_intr, msi_assign_cpu };
struct pic msix_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
msix_enable_intr, msi_vector, msix_source_pending,
NULL, NULL, msi_config_intr, msix_assign_cpu };
static int msi_enabled;
static struct sx msi_sx;
@ -162,17 +155,6 @@ msi_enable_intr(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
/*
* Since we can only enable the entire group at once, go ahead and
* enable the messages when the first message is given a handler.
* Note that we assume all devices will register a handler for the
* first message.
*/
if (msi->msi_index == 0) {
mtx_lock_spin(&icu_lock);
pci_enable_msi(msi->msi_dev, INTEL_ADDR(msi), INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
}
apic_enable_vector(msi->msi_vector);
}
@ -206,49 +188,11 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
msi->msi_cpu = apic_id;
if (bootverbose)
printf("msi: Assigning MSI IRQ %d to local APIC %u\n",
msi->msi_irq, msi->msi_cpu);
mtx_lock_spin(&icu_lock);
printf("msi: Assigning %s IRQ %d to local APIC %u\n",
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
msi->msi_cpu);
if (isrc->is_enabled)
pci_enable_msi(msi->msi_dev, INTEL_ADDR(msi), INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
}
static void
msix_enable_intr(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
mtx_lock_spin(&icu_lock);
pci_enable_msix(msi->msi_dev, msi->msi_index, INTEL_ADDR(msi),
INTEL_DATA(msi));
pci_unmask_msix(msi->msi_dev, msi->msi_index);
mtx_unlock_spin(&icu_lock);
apic_enable_vector(msi->msi_vector);
}
static int
msix_source_pending(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
return (pci_pending_msix(msi->msi_dev, msi->msi_index));
}
static void
msix_assign_cpu(struct intsrc *isrc, u_int apic_id)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
msi->msi_cpu = apic_id;
if (bootverbose)
printf("msi: Assigning MSI-X IRQ %d to local APIC %u\n",
msi->msi_irq, msi->msi_cpu);
mtx_lock_spin(&icu_lock);
if (isrc->is_enabled)
pci_enable_msix(msi->msi_dev, msi->msi_index, INTEL_ADDR(msi),
INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
}
void
@ -262,10 +206,21 @@ msi_init(void)
msi_enabled = 1;
intr_register_pic(&msi_pic);
intr_register_pic(&msix_pic);
sx_init(&msi_sx, "msi");
}
struct msi_intsrc *
msi_create_source(u_int irq)
{
struct msi_intsrc *msi;
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_irq = irq;
intr_register_source(&msi->msi_intsrc);
return (msi);
}
/*
* Try to allocate 'count' interrupt sources with contiguous IDT values. If
* we allocate any new sources, then their IRQ values will be at the end of
@ -317,14 +272,8 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
*newcount = count - cnt;
for (j = 0; j < *newcount; j++) {
/* Create a new MSI source. */
msi = malloc(sizeof(struct msi_intsrc), M_MSI,
M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_irq = i + j;
intr_register_source(&msi->msi_intsrc);
/* Add it to our array. */
/* Create a new MSI source and add it to our array. */
msi_create_source(i + j);
irqs[cnt] = i + j;
cnt++;
}
@ -344,13 +293,11 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
for (i = 0; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_dev = dev;
msi->msi_vector = vector + i;
if (bootverbose)
printf("msi: routing MSI IRQ %d to vector %u\n",
msi->msi_irq, msi->msi_vector);
msi->msi_index = i;
msi->msi_first = fsrc;
/* XXX: Somewhat gross. */
@ -395,8 +342,6 @@ msi_release(int *irqs, int count)
sx_xunlock(&msi_sx);
return (EINVAL);
}
KASSERT(first->msi_index == 0, ("index mismatch"));
KASSERT(first->msi_dev != NULL, ("unowned group"));
/* Clear all the extra messages in the group. */
@ -408,7 +353,6 @@ msi_release(int *irqs, int count)
msi->msi_dev = NULL;
apic_free_vector(msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_index = 0;
}
/* Clear out the first message. */
@ -423,7 +367,44 @@ msi_release(int *irqs, int count)
}
int
msix_alloc(device_t dev, int index, int *irq, int *new)
msi_map(int irq, uint64_t *addr, uint32_t *data)
{
struct msi_intsrc *msi;
sx_slock(&msi_sx);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
sx_sunlock(&msi_sx);
return (ENOENT);
}
/* Make sure this message is allocated to a device. */
if (msi->msi_dev == NULL) {
sx_sunlock(&msi_sx);
return (ENXIO);
}
/*
* If this message isn't an MSI-X message, make sure it's part
* of a gruop, and switch to the first message in the
* group.
*/
if (!msi->msi_msix) {
if (msi->msi_first == NULL) {
sx_sunlock(&msi_sx);
return (ENXIO);
}
msi = msi->msi_first;
}
*addr = INTEL_ADDR(msi);
*data = INTEL_DATA(msi);
sx_sunlock(&msi_sx);
return (0);
}
int
msix_alloc(device_t dev, int *irq, int *new)
{
struct msi_intsrc *msi;
int i, vector;
@ -457,11 +438,7 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
/* Create a new source. */
*new = 1;
msi = malloc(sizeof(struct msi_intsrc), M_MSI,
M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msix_pic;
msi->msi_irq = i;
intr_register_source(&msi->msi_intsrc);
msi = msi_create_source(i);
}
/* Allocate an IDT vector. */
@ -471,10 +448,8 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
vector);
/* Setup source. */
msi->msi_intsrc.is_pic = &msix_pic;
msi->msi_dev = dev;
msi->msi_vector = vector;
msi->msi_index = index;
msi->msi_msix = 1;
/* XXX: Somewhat gross. */
@ -485,30 +460,6 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
return (0);
}
int
msix_remap(int index, int irq)
{
struct msi_intsrc *msi;
sx_xlock(&msi_sx);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
sx_xunlock(&msi_sx);
return (ENOENT);
}
/* Make sure this is an MSI-X message. */
if (!msi->msi_msix) {
sx_xunlock(&msi_sx);
return (EINVAL);
}
KASSERT(msi->msi_dev != NULL, ("unowned message"));
msi->msi_index = index;
sx_xunlock(&msi_sx);
return (0);
}
int
msix_release(int irq)
{
@ -533,7 +484,6 @@ msix_release(int irq)
msi->msi_dev = NULL;
apic_free_vector(msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_index = 0;
msi->msi_msix = 0;
sx_xunlock(&msi_sx);

24
sys/amd64/amd64/nexus.c
View File

@ -105,9 +105,9 @@ static int nexus_get_resource(device_t, device_t, int, int, u_long *, u_long *);
static void nexus_delete_resource(device_t, device_t, int, int);
static int nexus_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs);
static int nexus_release_msi(device_t pcib, device_t dev, int count, int *irqs);
static int nexus_alloc_msix(device_t pcib, device_t dev, int index, int *irq);
static int nexus_remap_msix(device_t pcib, device_t dev, int index, int irq);
static int nexus_alloc_msix(device_t pcib, device_t dev, int *irq);
static int nexus_release_msix(device_t pcib, device_t dev, int irq);
static int nexus_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data);
static device_method_t nexus_methods[] = {
/* Device interface */
@ -137,8 +137,8 @@ static device_method_t nexus_methods[] = {
DEVMETHOD(pcib_alloc_msi, nexus_alloc_msi),
DEVMETHOD(pcib_release_msi, nexus_release_msi),
DEVMETHOD(pcib_alloc_msix, nexus_alloc_msix),
DEVMETHOD(pcib_remap_msix, nexus_remap_msix),
DEVMETHOD(pcib_release_msix, nexus_release_msix),
DEVMETHOD(pcib_map_msi, nexus_map_msi),
{ 0, 0 }
};
@ -520,23 +520,16 @@ nexus_delete_resource(device_t dev, device_t child, int type, int rid)
}
static int
nexus_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
nexus_alloc_msix(device_t pcib, device_t dev, int *irq)
{
int error, new;
error = msix_alloc(dev, index, irq, &new);
error = msix_alloc(dev, irq, &new);
if (new)
rman_manage_region(&irq_rman, *irq, *irq);
return (error);
}
static int
nexus_remap_msix(device_t pcib, device_t dev, int index, int irq)
{
return (msix_remap(index, irq));
}
static int
nexus_release_msix(device_t pcib, device_t dev, int irq)
{
@ -567,6 +560,13 @@ nexus_release_msi(device_t pcib, device_t dev, int count, int *irqs)
return (msi_release(irqs, count));
}
static int
nexus_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data)
{
return (msi_map(irq, addr, data));
}
#ifdef DEV_ISA
/*
* Placeholder which claims PnP 'devices' which describe system

4
sys/amd64/include/intr_machdep.h
View File

@ -150,9 +150,9 @@ void intrcnt_add(const char *name, u_long **countp);
int msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
int *newcount);
void msi_init(void);
int msi_map(int irq, uint64_t *addr, uint32_t *data);
int msi_release(int *irqs, int count);
int msix_alloc(device_t dev, int index, int *irq, int *new);
int msix_remap(int index, int irq);
int msix_alloc(device_t dev, int *irq, int *new);
int msix_release(int irq);
#endif /* !LOCORE */

18
sys/amd64/pci/pci_bus.c
View File

@ -81,7 +81,7 @@ legacy_pcib_route_interrupt(device_t pcib, device_t dev, int pin)
return (PCI_INVALID_IRQ);
}
/* Pass MSI alloc requests up to the nexus. */
/* Pass MSI requests up to the nexus. */
static int
legacy_pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
@ -95,12 +95,22 @@ legacy_pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
}
static int
legacy_pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
legacy_pcib_alloc_msix(device_t pcib, device_t dev, int *irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
static int
legacy_pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
static const char *
@ -347,8 +357,8 @@ static device_method_t legacy_pcib_methods[] = {
DEVMETHOD(pcib_alloc_msi, legacy_pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, legacy_pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, legacy_pcib_map_msi),
{ 0, 0 }
};

20
sys/dev/acpica/acpi_pcib_acpi.c
View File

@ -76,8 +76,10 @@ static int acpi_pcib_acpi_route_interrupt(device_t pcib,
device_t dev, int pin);
static int acpi_pcib_alloc_msi(device_t pcib, device_t dev,
int count, int maxcount, int *irqs);
static int acpi_pcib_map_msi(device_t pcib, device_t dev,
int irq, uint64_t *addr, uint32_t *data);
static int acpi_pcib_alloc_msix(device_t pcib, device_t dev,
int index, int *irq);
int *irq);
static struct resource *acpi_pcib_acpi_alloc_resource(device_t dev,
device_t child, int type, int *rid,
u_long start, u_long end, u_long count,
@ -110,8 +112,8 @@ static device_method_t acpi_pcib_acpi_methods[] = {
DEVMETHOD(pcib_alloc_msi, acpi_pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, acpi_pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, acpi_pcib_map_msi),
{0, 0}
};
@ -323,12 +325,22 @@ acpi_pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
}
static int
acpi_pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
acpi_pcib_alloc_msix(device_t pcib, device_t dev, int *irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
static int
acpi_pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
static u_long acpi_host_mem_start = 0x80000000;

2
sys/dev/acpica/acpi_pcib_pci.c
View File

@ -96,8 +96,8 @@ static device_method_t acpi_pcib_pci_methods[] = {
DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, pcib_map_msi),
{0, 0}
};

335
sys/dev/pci/pci.c
View File

@ -95,7 +95,16 @@ static int pci_modevent(module_t mod, int what, void *arg);
static void pci_hdrtypedata(device_t pcib, int b, int s, int f,
pcicfgregs *cfg);
static void pci_read_extcap(device_t pcib, pcicfgregs *cfg);
static void pci_disable_msi(device_t dev);
static void pci_enable_msi(device_t dev, uint64_t address,
uint16_t data);
static void pci_enable_msix(device_t dev, u_int index,
uint64_t address, uint32_t data);
static void pci_mask_msix(device_t dev, u_int index);
static void pci_unmask_msix(device_t dev, u_int index);
static int pci_msi_blacklisted(void);
static void pci_resume_msi(device_t dev);
static void pci_resume_msix(device_t dev);
static device_method_t pci_methods[] = {
/* Device interface */
@ -112,8 +121,8 @@ static device_method_t pci_methods[] = {
DEVMETHOD(bus_read_ivar, pci_read_ivar),
DEVMETHOD(bus_write_ivar, pci_write_ivar),
DEVMETHOD(bus_driver_added, pci_driver_added),
DEVMETHOD(bus_setup_intr, bus_generic_setup_intr),
DEVMETHOD(bus_teardown_intr, bus_generic_teardown_intr),
DEVMETHOD(bus_setup_intr, pci_setup_intr),
DEVMETHOD(bus_teardown_intr, pci_teardown_intr),
DEVMETHOD(bus_get_resource_list,pci_get_resource_list),
DEVMETHOD(bus_set_resource, bus_generic_rl_set_resource),
@ -663,7 +672,7 @@ pci_enable_msix(device_t dev, u_int index, uint64_t address, uint32_t data)
struct pcicfg_msix *msix = &dinfo->cfg.msix;
uint32_t offset;
KASSERT(msix->msix_alloc > index, ("bogus index"));
KASSERT(msix->msix_table_len > index, ("bogus index"));
offset = msix->msix_table_offset + index * 16;
bus_write_4(msix->msix_table_res, offset, address & 0xffffffff);
bus_write_4(msix->msix_table_res, offset + 4, address >> 32);
@ -693,7 +702,7 @@ pci_unmask_msix(device_t dev, u_int index)
struct pcicfg_msix *msix = &dinfo->cfg.msix;
uint32_t offset, val;
KASSERT(msix->msix_alloc > index, ("bogus index"));
KASSERT(msix->msix_table_len > index, ("bogus index"));
offset = msix->msix_table_offset + index * 16 + 12;
val = bus_read_4(msix->msix_table_res, offset);
if (val & PCIM_MSIX_VCTRL_MASK) {
@ -709,12 +718,45 @@ pci_pending_msix(device_t dev, u_int index)
struct pcicfg_msix *msix = &dinfo->cfg.msix;
uint32_t offset, bit;
KASSERT(msix->msix_alloc > index, ("bogus index"));
KASSERT(msix->msix_table_len > index, ("bogus index"));
offset = msix->msix_pba_offset + (index / 32) * 4;
bit = 1 << index % 32;
return (bus_read_4(msix->msix_pba_res, offset) & bit);
}
/*
* Restore MSI-X registers and table during resume. If MSI-X is
* enabled then walk the virtual table to restore the actual MSI-X
* table.
*/
static void
pci_resume_msix(device_t dev)
{
struct pci_devinfo *dinfo = device_get_ivars(dev);
struct pcicfg_msix *msix = &dinfo->cfg.msix;
struct msix_table_entry *mte;
struct msix_vector *mv;
int i;
if (msix->msix_alloc > 0) {
/* First, mask all vectors. */
for (i = 0; i < msix->msix_msgnum; i++)
pci_mask_msix(dev, i);
/* Second, program any messages with at least one handler. */
for (i = 0; i < msix->msix_table_len; i++) {
mte = &msix->msix_table[i];
if (mte->mte_vector == 0 || mte->mte_handlers == 0)
continue;
mv = &msix->msix_vectors[mte->mte_vector - 1];
pci_enable_msix(dev, i, mv->mv_address, mv->mv_data);
pci_unmask_msix(dev, i);
}
}
pci_write_config(dev, msix->msix_location + PCIR_MSIX_CTRL,
msix->msix_ctrl, 2);
}
/*
* Attempt to allocate *count MSI-X messages. The actual number allocated is
* returned in *count. After this function returns, each message will be
@ -772,8 +814,7 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count)
max = min(*count, cfg->msix.msix_msgnum);
for (i = 0; i < max; i++) {
/* Allocate a message. */
error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, i,
&irq);
error = PCIB_ALLOC_MSIX(device_get_parent(dev), child, &irq);
if (error)
break;
resource_list_add(&dinfo->resources, SYS_RES_IRQ, i + 1, irq,
@ -830,6 +871,17 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count)
for (i = 0; i < cfg->msix.msix_msgnum; i++)
pci_mask_msix(child, i);
/* Allocate and initialize vector data and virtual table. */
cfg->msix.msix_vectors = malloc(sizeof(struct msix_vector) * actual,
M_DEVBUF, M_WAITOK | M_ZERO);
cfg->msix.msix_table = malloc(sizeof(struct msix_table_entry) * actual,
M_DEVBUF, M_WAITOK | M_ZERO);
for (i = 0; i < actual; i++) {
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
cfg->msix.msix_vectors[i].mv_irq = rle->start;
cfg->msix.msix_table[i].mte_vector = i + 1;
}
/* Update control register to enable MSI-X. */
cfg->msix.msix_ctrl |= PCIM_MSIXCTRL_MSIX_ENABLE;
pci_write_config(child, cfg->msix.msix_location + PCIR_MSIX_CTRL,
@ -837,6 +889,7 @@ pci_alloc_msix_method(device_t dev, device_t child, int *count)
/* Update counts of alloc'd messages. */
cfg->msix.msix_alloc = actual;
cfg->msix.msix_table_len = actual;
*count = actual;
return (0);
}
@ -847,20 +900,22 @@ pci_release_msix(device_t dev, device_t child)
struct pci_devinfo *dinfo = device_get_ivars(child);
struct pcicfg_msix *msix = &dinfo->cfg.msix;
struct resource_list_entry *rle;
int count, i;
int i;
/* Do we have any messages to release? */
if (msix->msix_alloc == 0)
return (ENODEV);
/* Make sure none of the resources are allocated. */
for (i = 1, count = 0; count < msix->msix_alloc; i++) {
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i);
if (rle == NULL)
for (i = 0; i < msix->msix_table_len; i++) {
if (msix->msix_table[i].mte_vector == 0)
continue;
if (msix->msix_table[i].mte_handlers > 0)
return (EBUSY);
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
KASSERT(rle != NULL, ("missing resource"));
if (rle->res != NULL)
return (EBUSY);
count++;
}
/* Update control register to disable MSI-X. */
@ -868,18 +923,20 @@ pci_release_msix(device_t dev, device_t child)
pci_write_config(child, msix->msix_location + PCIR_MSIX_CTRL,
msix->msix_ctrl, 2);
/* Release the messages. */
for (i = 1, count = 0; count < msix->msix_alloc; i++) {
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i);
if (rle == NULL)
/* Free the resource list entries. */
for (i = 0; i < msix->msix_table_len; i++) {
if (msix->msix_table[i].mte_vector == 0)
continue;
PCIB_RELEASE_MSIX(device_get_parent(dev), child,
rle->start);
resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i);
count++;
resource_list_delete(&dinfo->resources, SYS_RES_IRQ, i + 1);
}
free(msix->msix_table, M_DEVBUF);
msix->msix_table_len = 0;
/* Update alloc count. */
/* Release the IRQs. */
for (i = 0; i < msix->msix_alloc; i++)
PCIB_RELEASE_MSIX(device_get_parent(dev), child,
msix->msix_vectors[i].mv_irq);
free(msix->msix_vectors, M_DEVBUF);
msix->msix_alloc = 0;
return (0);
}
@ -911,8 +968,6 @@ pci_enable_msi(device_t dev, uint64_t address, uint16_t data)
struct pcicfg_msi *msi = &dinfo->cfg.msi;
/* Write data and address values. */
msi->msi_addr = address;
msi->msi_data = data;
pci_write_config(dev, msi->msi_location + PCIR_MSI_ADDR,
address & 0xffffffff, 4);
if (msi->msi_ctrl & PCIM_MSICTRL_64BIT) {
@ -930,6 +985,18 @@ pci_enable_msi(device_t dev, uint64_t address, uint16_t data)
2);
}
void
pci_disable_msi(device_t dev)
{
struct pci_devinfo *dinfo = device_get_ivars(dev);
struct pcicfg_msi *msi = &dinfo->cfg.msi;
/* Disable MSI in the control register. */
msi->msi_ctrl &= ~PCIM_MSICTRL_MSI_ENABLE;
pci_write_config(dev, msi->msi_location + PCIR_MSI_CTRL, msi->msi_ctrl,
2);
}
/*
* Restore MSI registers during resume. If MSI is enabled then
* restore the data and address registers in addition to the control
@ -961,6 +1028,82 @@ pci_resume_msi(device_t dev)
2);
}
int
pci_remap_msi_irq(device_t dev, u_int irq)
{
struct pci_devinfo *dinfo = device_get_ivars(dev);
pcicfgregs *cfg = &dinfo->cfg;
struct resource_list_entry *rle;
struct msix_table_entry *mte;
struct msix_vector *mv;
device_t bus;
uint64_t addr;
uint32_t data;
int error, i, j;
bus = device_get_parent(dev);
/*
* Handle MSI first. We try to find this IRQ among our list
* of MSI IRQs. If we find it, we request updated address and
* data registers and apply the results.
*/
if (cfg->msi.msi_alloc > 0) {
/* If we don't have any active handlers, nothing to do. */
if (cfg->msi.msi_handlers == 0)
return (0);
for (i = 0; i < cfg->msi.msi_alloc; i++) {
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ,
i + 1);
if (rle->start == irq) {
error = PCIB_MAP_MSI(device_get_parent(bus),
dev, irq, &addr, &data);
if (error)
return (error);
pci_disable_msi(dev);
dinfo->cfg.msi.msi_addr = addr;
dinfo->cfg.msi.msi_data = data;
pci_enable_msi(dev, addr, data);
return (0);
}
}
return (ENOENT);
}
/*
* For MSI-X, we check to see if we have this IRQ. If we do,
* we request the updated mapping info. If that works, we go
* through all the slots that use this IRQ and update them.
*/
if (cfg->msix.msix_alloc > 0) {
for (i = 0; i < cfg->msix.msix_alloc; i++) {
mv = &cfg->msix.msix_vectors[i];
if (mv->mv_irq == irq) {
error = PCIB_MAP_MSI(device_get_parent(bus),
dev, irq, &addr, &data);
if (error)
return (error);
mv->mv_address = addr;
mv->mv_data = data;
for (j = 0; j < cfg->msix.msix_table_len; j++) {
mte = &cfg->msix.msix_table[j];
if (mte->mte_vector != i + 1)
continue;
if (mte->mte_handlers == 0)
continue;
pci_mask_msix(dev, j);
pci_enable_msix(dev, j, addr, data);
pci_unmask_msix(dev, j);
}
}
}
return (ENOENT);
}
return (ENOENT);
}
/*
* Returns true if the specified device is blacklisted because MSI
* doesn't work.
@ -1125,6 +1268,7 @@ pci_alloc_msi_method(device_t dev, device_t child, int *count)
/* Update counts of alloc'd messages. */
cfg->msi.msi_alloc = actual;
cfg->msi.msi_handlers = 0;
*count = actual;
return (0);
}
@ -1149,6 +1293,8 @@ pci_release_msi_method(device_t dev, device_t child)
KASSERT(msi->msi_alloc <= 32, ("more than 32 alloc'd messages"));
/* Make sure none of the resources are allocated. */
if (msi->msi_handlers > 0)
return (EBUSY);
for (i = 0; i < msi->msi_alloc; i++) {
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, i + 1);
KASSERT(rle != NULL, ("missing MSI resource"));
@ -1157,8 +1303,10 @@ pci_release_msi_method(device_t dev, device_t child)
irqs[i] = rle->start;
}
/* Update control register with 0 count and disable MSI. */
msi->msi_ctrl &= ~(PCIM_MSICTRL_MME_MASK | PCIM_MSICTRL_MSI_ENABLE);
/* Update control register with 0 count. */
KASSERT(!(msi->msi_ctrl & PCIM_MSICTRL_MSI_ENABLE),
("%s: MSI still enabled", __func__));
msi->msi_ctrl &= ~PCIM_MSICTRL_MME_MASK;
pci_write_config(child, msi->msi_location + PCIR_MSI_CTRL,
msi->msi_ctrl, 2);
@ -1169,6 +1317,8 @@ pci_release_msi_method(device_t dev, device_t child)
/* Update alloc count. */
msi->msi_alloc = 0;
msi->msi_addr = 0;
msi->msi_data = 0;
return (0);
}
@ -1982,6 +2132,134 @@ pci_driver_added(device_t dev, driver_t *driver)
free(devlist, M_TEMP);
}
int
pci_setup_intr(device_t dev, device_t child, struct resource *irq, int flags,
driver_intr_t *intr, void *arg, void **cookiep)
{
struct pci_devinfo *dinfo;
struct msix_table_entry *mte;
struct msix_vector *mv;
uint64_t addr;
uint32_t data;
void *cookie;
int error, rid;
error = bus_generic_setup_intr(dev, child, irq, flags, intr, arg,
&cookie);
if (error)
return (error);
/*
* If this is a direct child, check to see if the interrupt is
* MSI or MSI-X. If so, ask our parent to map the MSI and give
* us the address and data register values. If we fail for some
* reason, teardown the interrupt handler.
*/
rid = rman_get_rid(irq);
if (device_get_parent(child) == dev && rid > 0) {
dinfo = device_get_ivars(child);
if (dinfo->cfg.msi.msi_alloc > 0) {
if (dinfo->cfg.msi.msi_addr == 0) {
KASSERT(dinfo->cfg.msi.msi_handlers == 0,
("MSI has handlers, but vectors not mapped"));
error = PCIB_MAP_MSI(device_get_parent(dev),
child, rman_get_start(irq), &addr, &data);
if (error)
goto bad;
dinfo->cfg.msi.msi_addr = addr;
dinfo->cfg.msi.msi_data = data;
pci_enable_msi(child, addr, data);
}
dinfo->cfg.msi.msi_handlers++;
} else {
KASSERT(dinfo->cfg.msix.msix_alloc > 0,
("No MSI or MSI-X interrupts allocated"));
KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
("MSI-X index too high"));
mte = &dinfo->cfg.msix.msix_table[rid - 1];
KASSERT(mte->mte_vector != 0, ("no message vector"));
mv = &dinfo->cfg.msix.msix_vectors[mte->mte_vector - 1];
KASSERT(mv->mv_irq == rman_get_start(irq),
("IRQ mismatch"));
if (mv->mv_address == 0) {
KASSERT(mte->mte_handlers == 0,
("MSI-X table entry has handlers, but vector not mapped"));
error = PCIB_MAP_MSI(device_get_parent(dev),
child, rman_get_start(irq), &addr, &data);
if (error)
goto bad;
mv->mv_address = addr;
mv->mv_data = data;
}
if (mte->mte_handlers == 0) {
pci_enable_msix(child, rid - 1, mv->mv_address,
mv->mv_data);
pci_unmask_msix(child, rid - 1);
}
mte->mte_handlers++;
}
bad:
if (error) {
(void)bus_generic_teardown_intr(dev, child, irq,
cookie);
return (error);
}
}
*cookiep = cookie;
return (0);
}
int
pci_teardown_intr(device_t dev, device_t child, struct resource *irq,
void *cookie)
{
struct msix_table_entry *mte;
struct resource_list_entry *rle;
struct pci_devinfo *dinfo;
int error, rid;
/*
* If this is a direct child, check to see if the interrupt is
* MSI or MSI-X. If so, decrement the appropriate handlers
* count and mask the MSI-X message, or disable MSI messages
* if the count drops to 0.
*/
if (irq == NULL || !(rman_get_flags(irq) & RF_ACTIVE))
return (EINVAL);
rid = rman_get_rid(irq);
if (device_get_parent(child) == dev && rid > 0) {
dinfo = device_get_ivars(child);
rle = resource_list_find(&dinfo->resources, SYS_RES_IRQ, rid);
if (rle->res != irq)
return (EINVAL);
if (dinfo->cfg.msi.msi_alloc > 0) {
KASSERT(rid <= dinfo->cfg.msi.msi_alloc,
("MSI-X index too high"));
if (dinfo->cfg.msi.msi_handlers == 0)
return (EINVAL);
dinfo->cfg.msi.msi_handlers--;
if (dinfo->cfg.msi.msi_handlers == 0)
pci_disable_msi(child);
} else {
KASSERT(dinfo->cfg.msix.msix_alloc > 0,
("No MSI or MSI-X interrupts allocated"));
KASSERT(rid <= dinfo->cfg.msix.msix_table_len,
("MSI-X index too high"));
mte = &dinfo->cfg.msix.msix_table[rid - 1];
if (mte->mte_handlers == 0)
return (EINVAL);
mte->mte_handlers--;
if (mte->mte_handlers == 0)
pci_mask_msix(child, rid - 1);
}
}
error = bus_generic_teardown_intr(dev, child, irq, cookie);
if (device_get_parent(child) == dev && rid > 0)
KASSERT(error == 0,
("%s: generic teardown failed for MSI/MSI-X", __func__));
return (error);
}
int
pci_print_child(device_t dev, device_t child)
{
@ -2753,12 +3031,11 @@ pci_cfg_restore(device_t dev, struct pci_devinfo *dinfo)
pci_write_config(dev, PCIR_PROGIF, dinfo->cfg.progif, 1);
pci_write_config(dev, PCIR_REVID, dinfo->cfg.revid, 1);
/*
* Restore MSI configuration if it is present. If MSI is enabled,
* then restore the data and addr registers.
*/
/* Restore MSI and MSI-X configurations if they are present. */
if (dinfo->cfg.msi.msi_location != 0)
pci_resume_msi(dev);
if (dinfo->cfg.msix.msix_location != 0)
pci_resume_msix(dev);
}
void

31
sys/dev/pci/pci_pci.c
View File

@ -82,8 +82,8 @@ static device_method_t pcib_methods[] = {
DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, pcib_map_msi),
{ 0, 0 }
};
@ -541,7 +541,7 @@ pcib_route_interrupt(device_t pcib, device_t dev, int pin)
return(intnum);
}
/* Pass request to alloc MSI messages up to the parent bridge. */
/* Pass request to alloc MSI/MSI-X messages up to the parent bridge. */
int
pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs)
{
@ -555,7 +555,7 @@ pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs)
irqs));
}
/* Pass request to release MSI messages up to the parent bridge. */
/* Pass request to release MSI/MSI-X messages up to the parent bridge. */
int
pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs)
{
@ -567,7 +567,7 @@ pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs)
/* Pass request to alloc an MSI-X message up to the parent bridge. */
int
pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
pcib_alloc_msix(device_t pcib, device_t dev, int *irq)
{
struct pcib_softc *sc = device_get_softc(pcib);
device_t bus;
@ -575,17 +575,7 @@ pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
if (sc->flags & PCIB_DISABLE_MSI)
return (ENXIO);
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
}
/* Pass request to remap an MSI-X message up to the parent bridge. */
int
pcib_remap_msix(device_t pcib, device_t dev, int index, int irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_REMAP_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
/* Pass request to release an MSI-X message up to the parent bridge. */
@ -598,6 +588,17 @@ pcib_release_msix(device_t pcib, device_t dev, int irq)
return (PCIB_RELEASE_MSIX(device_get_parent(bus), dev, irq));
}
/* Pass request to map MSI/MSI-X message up to parent bridge. */
int
pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
/*
* Try to read the bus number of a host-PCI bridge using appropriate config
* registers.

5
sys/dev/pci/pci_private.h
View File

@ -49,6 +49,11 @@ int pci_read_ivar(device_t dev, device_t child, int which,
uintptr_t *result);
int pci_write_ivar(device_t dev, device_t child, int which,
uintptr_t value);
int pci_setup_intr(device_t dev, device_t child,
struct resource *irq, int flags, driver_intr_t *intr,
void *arg, void **cookiep);
int pci_teardown_intr(device_t dev, device_t child,
struct resource *irq, void *cookie);
int pci_set_powerstate_method(device_t dev, device_t child,
int state);
int pci_get_powerstate_method(device_t dev, device_t child);

27
sys/dev/pci/pcib_if.m
View File

@ -105,8 +105,8 @@ METHOD int alloc_msi {
};
#
# Release 'count' MSI message mapped onto 'count' IRQs stored in the
# array pointed to by 'irq'.
# Release 'count' MSI messages mapped onto 'count' IRQs stored in the
# array pointed to by 'irqs'.
#
METHOD int release_msi {
device_t pcib;
@ -121,20 +121,9 @@ METHOD int release_msi {
METHOD int alloc_msix {
device_t pcib;
device_t dev;
int index;
int *irq;
};
#
# Remap a single MSI-X message to a different index.
#
METHOD int remap_msix {
device_t pcib;
device_t dev;
int index;
int irq;
};
#
# Release a single MSI-X message mapped onto 'irq'.
#
@ -143,3 +132,15 @@ METHOD int release_msix {
device_t dev;
int irq;
};
#
# Determine the MSI/MSI-X message address and data for 'irq'. The address
# is returned in '*addr', and the data in '*data'.
#
METHOD int map_msi {
device_t pcib;
device_t dev;
int irq;
uint64_t *addr;
uint32_t *data;
};

4
sys/dev/pci/pcib_private.h
View File

@ -77,8 +77,8 @@ void pcib_write_config(device_t dev, int b, int s, int f, int reg, uint32_t val
int pcib_route_interrupt(device_t pcib, device_t dev, int pin);
int pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs);
int pcib_release_msi(device_t pcib, device_t dev, int count, int *irqs);
int pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq);
int pcib_remap_msix(device_t pcib, device_t dev, int index, int irq);
int pcib_alloc_msix(device_t pcib, device_t dev, int *irq);
int pcib_release_msix(device_t pcib, device_t dev, int irq);
int pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data);
#endif

33
sys/dev/pci/pcivar.h
View File

@ -67,18 +67,33 @@ struct pcicfg_msi {
int msi_alloc; /* Number of allocated messages. */
uint64_t msi_addr; /* Contents of address register. */
uint16_t msi_data; /* Contents of data register. */
u_int msi_handlers;
};
/* Interesting values for PCI MSI-X */
struct msix_vector {
uint64_t mv_address; /* Contents of address register. */
uint32_t mv_data; /* Contents of data register. */
int mv_irq;
};
struct msix_table_entry {
u_int mte_vector; /* 1-based index into msix_vectors array. */
u_int mte_handlers;
};
struct pcicfg_msix {
uint16_t msix_ctrl; /* Message Control */
uint8_t msix_location; /* Offset of MSI-X capability registers. */
uint16_t msix_msgnum; /* Number of messages */
int msix_alloc; /* Number of allocated messages. */
uint8_t msix_location; /* Offset of MSI-X capability registers. */
uint8_t msix_table_bar; /* BAR containing vector table. */
uint8_t msix_pba_bar; /* BAR containing PBA. */
uint32_t msix_table_offset;
uint32_t msix_pba_offset;
int msix_alloc; /* Number of allocated vectors. */
int msix_table_len; /* Length of virtual table. */
struct msix_table_entry *msix_table; /* Virtual table. */
struct msix_vector *msix_vectors; /* Array of allocated vectors. */
struct resource *msix_table_res; /* Resource containing vector table. */
struct resource *msix_pba_res; /* Resource containing PBA. */
};
@ -400,13 +415,15 @@ pci_msix_count(device_t dev)
device_t pci_find_bsf(uint8_t, uint8_t, uint8_t);
device_t pci_find_device(uint16_t, uint16_t);
/* Used by MD code to program MSI and MSI-X registers. */
void pci_enable_msi(device_t dev, uint64_t address, uint16_t data);
void pci_enable_msix(device_t dev, u_int index, uint64_t address,
uint32_t data);
void pci_mask_msix(device_t dev, u_int index);
/*
* Can be used by MD code to request the PCI bus to re-map an MSI or
* MSI-X message.
*/
int pci_remap_msi_irq(device_t dev, u_int irq);
/* Can be used by drivers to manage the MSI-X table. */
int pci_pending_msix(device_t dev, u_int index);
void pci_unmask_msix(device_t dev, u_int index);
int pci_msi_device_blacklisted(device_t dev);
#endif /* _SYS_BUS_H_ */

20
sys/i386/i386/mptable_pci.c
View File

@ -72,7 +72,7 @@ mptable_hostb_attach(device_t dev)
return (bus_generic_attach(dev));
}
/* Pass MSI alloc requests up to the nexus. */
/* Pass MSI requests up to the nexus. */
static int
mptable_hostb_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
int *irqs)
@ -85,12 +85,22 @@ mptable_hostb_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
}
static int
mptable_hostb_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
mptable_hostb_alloc_msix(device_t pcib, device_t dev, int *irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
static int
mptable_hostb_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
static device_method_t mptable_hostb_methods[] = {
@ -120,8 +130,8 @@ static device_method_t mptable_hostb_methods[] = {
DEVMETHOD(pcib_alloc_msi, mptable_hostb_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, mptable_hostb_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, mptable_hostb_map_msi),
{ 0, 0 }
};
@ -177,8 +187,8 @@ static device_method_t mptable_pcib_pci_methods[] = {
DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, pcib_map_msi),
{0, 0}
};

168
sys/i386/i386/msi.c
View File

@ -99,21 +99,20 @@ static MALLOC_DEFINE(M_MSI, "msi", "PCI MSI");
* assigned an ID by the system; however, a group will use the ID from
* the first message.
*
* For MSI-X, each message is isolated, and msi_index indicates the
* index of this message in the device's MSI-X table.
* For MSI-X, each message is isolated.
*/
struct msi_intsrc {
struct intsrc msi_intsrc;
device_t msi_dev; /* Owning device. (g) */
struct msi_intsrc *msi_first; /* First source in group. */
u_int msi_irq; /* IRQ cookie. */
u_int msi_index; /* Index of this message. */
u_int msi_msix; /* MSI-X message. */
u_int msi_vector:8; /* IDT vector. */
u_int msi_cpu:8; /* Local APIC ID. (g) */
u_int msi_count:8; /* Messages in this group. (g) */
};
static struct msi_intsrc *msi_create_source(u_int irq);
static void msi_enable_source(struct intsrc *isrc);
static void msi_disable_source(struct intsrc *isrc, int eoi);
static void msi_eoi_source(struct intsrc *isrc);
@ -123,16 +122,10 @@ static int msi_source_pending(struct intsrc *isrc);
static int msi_config_intr(struct intsrc *isrc, enum intr_trigger trig,
enum intr_polarity pol);
static void msi_assign_cpu(struct intsrc *isrc, u_int apic_id);
static void msix_enable_intr(struct intsrc *isrc);
static int msix_source_pending(struct intsrc *isrc);
static void msix_assign_cpu(struct intsrc *isrc, u_int apic_id);
struct pic msi_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
msi_enable_intr, msi_vector, msi_source_pending,
NULL, NULL, msi_config_intr, msi_assign_cpu };
struct pic msix_pic = { msi_enable_source, msi_disable_source, msi_eoi_source,
msix_enable_intr, msi_vector, msix_source_pending,
NULL, NULL, msi_config_intr, msix_assign_cpu };
static int msi_enabled;
static struct sx msi_sx;
@ -162,17 +155,6 @@ msi_enable_intr(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
/*
* Since we can only enable the entire group at once, go ahead and
* enable the messages when the first message is given a handler.
* Note that we assume all devices will register a handler for the
* first message.
*/
if (msi->msi_index == 0) {
mtx_lock_spin(&icu_lock);
pci_enable_msi(msi->msi_dev, INTEL_ADDR(msi), INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
}
apic_enable_vector(msi->msi_vector);
}
@ -206,49 +188,11 @@ msi_assign_cpu(struct intsrc *isrc, u_int apic_id)
msi->msi_cpu = apic_id;
if (bootverbose)
printf("msi: Assigning MSI IRQ %d to local APIC %u\n",
msi->msi_irq, msi->msi_cpu);
mtx_lock_spin(&icu_lock);
printf("msi: Assigning %s IRQ %d to local APIC %u\n",
msi->msi_msix ? "MSI-X" : "MSI", msi->msi_irq,
msi->msi_cpu);
if (isrc->is_enabled)
pci_enable_msi(msi->msi_dev, INTEL_ADDR(msi), INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
}
static void
msix_enable_intr(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
mtx_lock_spin(&icu_lock);
pci_enable_msix(msi->msi_dev, msi->msi_index, INTEL_ADDR(msi),
INTEL_DATA(msi));
pci_unmask_msix(msi->msi_dev, msi->msi_index);
mtx_unlock_spin(&icu_lock);
apic_enable_vector(msi->msi_vector);
}
static int
msix_source_pending(struct intsrc *isrc)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
return (pci_pending_msix(msi->msi_dev, msi->msi_index));
}
static void
msix_assign_cpu(struct intsrc *isrc, u_int apic_id)
{
struct msi_intsrc *msi = (struct msi_intsrc *)isrc;
msi->msi_cpu = apic_id;
if (bootverbose)
printf("msi: Assigning MSI-X IRQ %d to local APIC %u\n",
msi->msi_irq, msi->msi_cpu);
mtx_lock_spin(&icu_lock);
if (isrc->is_enabled)
pci_enable_msix(msi->msi_dev, msi->msi_index, INTEL_ADDR(msi),
INTEL_DATA(msi));
mtx_unlock_spin(&icu_lock);
pci_remap_msi_irq(msi->msi_dev, msi->msi_irq);
}
void
@ -262,10 +206,21 @@ msi_init(void)
msi_enabled = 1;
intr_register_pic(&msi_pic);
intr_register_pic(&msix_pic);
sx_init(&msi_sx, "msi");
}
struct msi_intsrc *
msi_create_source(u_int irq)
{
struct msi_intsrc *msi;
msi = malloc(sizeof(struct msi_intsrc), M_MSI, M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_irq = irq;
intr_register_source(&msi->msi_intsrc);
return (msi);
}
/*
* Try to allocate 'count' interrupt sources with contiguous IDT values. If
* we allocate any new sources, then their IRQ values will be at the end of
@ -317,14 +272,8 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
*newcount = count - cnt;
for (j = 0; j < *newcount; j++) {
/* Create a new MSI source. */
msi = malloc(sizeof(struct msi_intsrc), M_MSI,
M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_irq = i + j;
intr_register_source(&msi->msi_intsrc);
/* Add it to our array. */
/* Create a new MSI source and add it to our array. */
msi_create_source(i + j);
irqs[cnt] = i + j;
cnt++;
}
@ -344,13 +293,11 @@ msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
fsrc = (struct msi_intsrc *)intr_lookup_source(irqs[0]);
for (i = 0; i < count; i++) {
msi = (struct msi_intsrc *)intr_lookup_source(irqs[i]);
msi->msi_intsrc.is_pic = &msi_pic;
msi->msi_dev = dev;
msi->msi_vector = vector + i;
if (bootverbose)
printf("msi: routing MSI IRQ %d to vector %u\n",
msi->msi_irq, msi->msi_vector);
msi->msi_index = i;
msi->msi_first = fsrc;
/* XXX: Somewhat gross. */
@ -395,8 +342,6 @@ msi_release(int *irqs, int count)
sx_xunlock(&msi_sx);
return (EINVAL);
}
KASSERT(first->msi_index == 0, ("index mismatch"));
KASSERT(first->msi_dev != NULL, ("unowned group"));
/* Clear all the extra messages in the group. */
@ -408,7 +353,6 @@ msi_release(int *irqs, int count)
msi->msi_dev = NULL;
apic_free_vector(msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_index = 0;
}
/* Clear out the first message. */
@ -423,7 +367,44 @@ msi_release(int *irqs, int count)
}
int
msix_alloc(device_t dev, int index, int *irq, int *new)
msi_map(int irq, uint64_t *addr, uint32_t *data)
{
struct msi_intsrc *msi;
sx_slock(&msi_sx);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
sx_sunlock(&msi_sx);
return (ENOENT);
}
/* Make sure this message is allocated to a device. */
if (msi->msi_dev == NULL) {
sx_sunlock(&msi_sx);
return (ENXIO);
}
/*
* If this message isn't an MSI-X message, make sure it's part
* of a gruop, and switch to the first message in the
* group.
*/
if (!msi->msi_msix) {
if (msi->msi_first == NULL) {
sx_sunlock(&msi_sx);
return (ENXIO);
}
msi = msi->msi_first;
}
*addr = INTEL_ADDR(msi);
*data = INTEL_DATA(msi);
sx_sunlock(&msi_sx);
return (0);
}
int
msix_alloc(device_t dev, int *irq, int *new)
{
struct msi_intsrc *msi;
int i, vector;
@ -457,11 +438,7 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
/* Create a new source. */
*new = 1;
msi = malloc(sizeof(struct msi_intsrc), M_MSI,
M_WAITOK | M_ZERO);
msi->msi_intsrc.is_pic = &msix_pic;
msi->msi_irq = i;
intr_register_source(&msi->msi_intsrc);
msi = msi_create_source(i);
}
/* Allocate an IDT vector. */
@ -471,10 +448,8 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
vector);
/* Setup source. */
msi->msi_intsrc.is_pic = &msix_pic;
msi->msi_dev = dev;
msi->msi_vector = vector;
msi->msi_index = index;
msi->msi_msix = 1;
/* XXX: Somewhat gross. */
@ -485,30 +460,6 @@ msix_alloc(device_t dev, int index, int *irq, int *new)
return (0);
}
int
msix_remap(int index, int irq)
{
struct msi_intsrc *msi;
sx_xlock(&msi_sx);
msi = (struct msi_intsrc *)intr_lookup_source(irq);
if (msi == NULL) {
sx_xunlock(&msi_sx);
return (ENOENT);
}
/* Make sure this is an MSI-X message. */
if (!msi->msi_msix) {
sx_xunlock(&msi_sx);
return (EINVAL);
}
KASSERT(msi->msi_dev != NULL, ("unowned message"));
msi->msi_index = index;
sx_xunlock(&msi_sx);
return (0);
}
int
msix_release(int irq)
{
@ -533,7 +484,6 @@ msix_release(int irq)
msi->msi_dev = NULL;
apic_free_vector(msi->msi_vector, msi->msi_irq);
msi->msi_vector = 0;
msi->msi_index = 0;
msi->msi_msix = 0;
sx_xunlock(&msi_sx);

24
sys/i386/i386/nexus.c
View File

@ -112,9 +112,9 @@ static void nexus_delete_resource(device_t, device_t, int, int);
#ifdef DEV_APIC
static int nexus_alloc_msi(device_t pcib, device_t dev, int count, int maxcount, int *irqs);
static int nexus_release_msi(device_t pcib, device_t dev, int count, int *irqs);
static int nexus_alloc_msix(device_t pcib, device_t dev, int index, int *irq);
static int nexus_remap_msix(device_t pcib, device_t dev, int index, int irq);
static int nexus_alloc_msix(device_t pcib, device_t dev, int *irq);
static int nexus_release_msix(device_t pcib, device_t dev, int irq);
static int nexus_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data);
#endif
static device_method_t nexus_methods[] = {
@ -146,8 +146,8 @@ static device_method_t nexus_methods[] = {
DEVMETHOD(pcib_alloc_msi, nexus_alloc_msi),
DEVMETHOD(pcib_release_msi, nexus_release_msi),
DEVMETHOD(pcib_alloc_msix, nexus_alloc_msix),
DEVMETHOD(pcib_remap_msix, nexus_remap_msix),
DEVMETHOD(pcib_release_msix, nexus_release_msix),
DEVMETHOD(pcib_map_msi, nexus_map_msi),
#endif
{ 0, 0 }
@ -575,23 +575,16 @@ nexus_delete_resource(device_t dev, device_t child, int type, int rid)
#ifdef DEV_APIC
static int
nexus_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
nexus_alloc_msix(device_t pcib, device_t dev, int *irq)
{
int error, new;
error = msix_alloc(dev, index, irq, &new);
error = msix_alloc(dev, irq, &new);
if (new)
rman_manage_region(&irq_rman, *irq, *irq);
return (error);
}
static int
nexus_remap_msix(device_t pcib, device_t dev, int index, int irq)
{
return (msix_remap(index, irq));
}
static int
nexus_release_msix(device_t pcib, device_t dev, int irq)
{
@ -621,6 +614,13 @@ nexus_release_msi(device_t pcib, device_t dev, int count, int *irqs)
return (msi_release(irqs, count));
}
static int
nexus_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr, uint32_t *data)
{
return (msi_map(irq, addr, data));
}
#endif
#ifdef DEV_ISA

4
sys/i386/include/intr_machdep.h
View File

@ -147,9 +147,9 @@ void intrcnt_add(const char *name, u_long **countp);
int msi_alloc(device_t dev, int count, int maxcount, int *irqs, int *newirq,
int *newcount);
void msi_init(void);
int msi_map(int irq, uint64_t *addr, uint32_t *data);
int msi_release(int* irqs, int count);
int msix_alloc(device_t dev, int index, int *irq, int *new);
int msix_remap(int index, int irq);
int msix_alloc(device_t dev, int *irq, int *new);
int msix_release(int irq);
#endif /* !LOCORE */

20
sys/i386/pci/pci_bus.c
View File

@ -77,7 +77,7 @@ legacy_pcib_write_config(device_t dev, int bus, int slot, int func,
pci_cfgregwrite(bus, slot, func, reg, data, bytes);
}
/* Pass MSI alloc requests up to the nexus. */
/* Pass MSI requests up to the nexus. */
static int
legacy_pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
@ -91,12 +91,22 @@ legacy_pcib_alloc_msi(device_t pcib, device_t dev, int count, int maxcount,
}
static int
legacy_pcib_alloc_msix(device_t pcib, device_t dev, int index, int *irq)
legacy_pcib_alloc_msix(device_t pcib, device_t dev, int *irq)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, index, irq));
return (PCIB_ALLOC_MSIX(device_get_parent(bus), dev, irq));
}
static int
legacy_pcib_map_msi(device_t pcib, device_t dev, int irq, uint64_t *addr,
uint32_t *data)
{
device_t bus;
bus = device_get_parent(pcib);
return (PCIB_MAP_MSI(device_get_parent(bus), dev, irq, addr, data));
}
static const char *
@ -559,8 +569,8 @@ static device_method_t legacy_pcib_methods[] = {
DEVMETHOD(pcib_alloc_msi, legacy_pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, legacy_pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, legacy_pcib_map_msi),
{ 0, 0 }
};
@ -711,8 +721,8 @@ static device_method_t pcibios_pcib_pci_methods[] = {
DEVMETHOD(pcib_alloc_msi, pcib_alloc_msi),
DEVMETHOD(pcib_release_msi, pcib_release_msi),
DEVMETHOD(pcib_alloc_msix, pcib_alloc_msix),
DEVMETHOD(pcib_remap_msix, pcib_remap_msix),
DEVMETHOD(pcib_release_msix, pcib_release_msix),
DEVMETHOD(pcib_map_msi, pcib_map_msi),
{0, 0}
};