Update the I/O MMU in bhyve when PCI devices are added and removed.
When the I/O MMU is active in bhyve, all PCI devices need valid entries in the DMAR context tables. The I/O MMU code does a single enumeration of the available PCI devices during initialization to add all existing devices to a domain representing the host. The ppt(4) driver then moves pass through devices in and out of domains for virtual machines as needed. However, when new PCI devices were added at runtime either via SR-IOV or HotPlug, the I/O MMU tables were not updated. This change adds a new set of EVENTHANDLERS that are invoked when PCI devices are added and deleted. The I/O MMU driver in bhyve installs handlers for these events which it uses to add and remove devices to the "host" domain. Reviewed by: imp Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D7667
This commit is contained in:
parent
a4554c3736
commit
64414cc00f
Notes:
svn2git
2020-12-20 02:59:44 +00:00
svn path=/head/; revision=305497
@ -25,7 +25,7 @@
|
||||
.\"
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd September 1, 2016
|
||||
.Dd September 6, 2016
|
||||
.Dt PCI 9
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -149,6 +149,10 @@
|
||||
.Fn pcie_read_config "device_t dev" "int reg" "int width"
|
||||
.Ft void
|
||||
.Fn pcie_write_config "device_t dev" "int reg" "uint32_t val" "int width"
|
||||
.Ft void
|
||||
.Fn pci_event_fn "void *arg" "device_t dev"
|
||||
.Fn EVENTHANDLER_REGISTER "pci_add_device" "pci_event_fn"
|
||||
.Fn EVENTHANDLER_DEREGISTER "pci_delete_resource" "pci_event_fn"
|
||||
.In dev/pci/pci_iov.h
|
||||
.Ft int
|
||||
.Fn pci_iov_attach "device_t dev" "nvlist_t *pf_schema" "nvlist_t *vf_schema"
|
||||
@ -910,6 +914,24 @@ with one in the new distribution.
|
||||
The
|
||||
.Fn pci_remap_msix
|
||||
function will fail if this condition is not met.
|
||||
.Ss Device Events
|
||||
The
|
||||
.Va pci_add_device
|
||||
event handler is invoked every time a new PCI device is added to the system.
|
||||
This includes the creation of Virtual Functions via SR-IOV.
|
||||
.Pp
|
||||
The
|
||||
.Va pci_delete_device
|
||||
event handler is invoked every time a PCI device is removed from the system.
|
||||
.Pp
|
||||
Both event handlers pass the
|
||||
.Vt device_t
|
||||
object of the relevant PCI device as
|
||||
.Fa dev
|
||||
to each callback function.
|
||||
Both event handlers are invoked while
|
||||
.Fa dev
|
||||
is unattached but with valid instance variables.
|
||||
.Sh SEE ALSO
|
||||
.Xr pci 4 ,
|
||||
.Xr pciconf 8 ,
|
||||
@ -921,6 +943,7 @@ function will fail if this condition is not met.
|
||||
.Xr devclass 9 ,
|
||||
.Xr device 9 ,
|
||||
.Xr driver 9 ,
|
||||
.Xr eventhandler 9 ,
|
||||
.Xr rman 9
|
||||
.Rs
|
||||
.%B FreeBSD Developers' Handbook
|
||||
|
@ -58,6 +58,7 @@ SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
|
||||
|
||||
static struct iommu_ops *ops;
|
||||
static void *host_domain;
|
||||
static eventhandler_tag add_tag, delete_tag;
|
||||
|
||||
static __inline int
|
||||
IOMMU_INIT(void)
|
||||
@ -153,6 +154,21 @@ IOMMU_DISABLE(void)
|
||||
(*ops->disable)();
|
||||
}
|
||||
|
||||
static void
|
||||
iommu_pci_add(void *arg, device_t dev)
|
||||
{
|
||||
|
||||
/* Add new devices to the host domain. */
|
||||
iommu_add_device(host_domain, pci_get_rid(dev));
|
||||
}
|
||||
|
||||
static void
|
||||
iommu_pci_delete(void *arg, device_t dev)
|
||||
{
|
||||
|
||||
iommu_remove_device(host_domain, pci_get_rid(dev));
|
||||
}
|
||||
|
||||
static void
|
||||
iommu_init(void)
|
||||
{
|
||||
@ -195,6 +211,9 @@ iommu_init(void)
|
||||
*/
|
||||
iommu_create_mapping(host_domain, 0, 0, maxaddr);
|
||||
|
||||
add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
|
||||
delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
|
||||
NULL, 0);
|
||||
for (bus = 0; bus <= PCI_BUSMAX; bus++) {
|
||||
for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
|
||||
for (func = 0; func <= PCI_FUNCMAX; func++) {
|
||||
@ -215,6 +234,15 @@ iommu_init(void)
|
||||
void
|
||||
iommu_cleanup(void)
|
||||
{
|
||||
|
||||
if (add_tag != NULL) {
|
||||
EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
|
||||
add_tag = NULL;
|
||||
}
|
||||
if (delete_tag != NULL) {
|
||||
EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
|
||||
delete_tag = NULL;
|
||||
}
|
||||
IOMMU_DISABLE();
|
||||
IOMMU_DESTROY_DOMAIN(host_domain);
|
||||
IOMMU_CLEANUP();
|
||||
|
@ -4071,6 +4071,7 @@ pci_add_child(device_t bus, struct pci_devinfo *dinfo)
|
||||
pci_print_verbose(dinfo);
|
||||
pci_add_resources(bus, dinfo->cfg.dev, 0, 0);
|
||||
pci_child_added(dinfo->cfg.dev);
|
||||
EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
|
||||
}
|
||||
|
||||
void
|
||||
@ -5312,6 +5313,8 @@ pci_child_deleted(device_t dev, device_t child)
|
||||
dinfo = device_get_ivars(child);
|
||||
rl = &dinfo->resources;
|
||||
|
||||
EVENTHANDLER_INVOKE(pci_delete_device, child);
|
||||
|
||||
/* Turn off access to resources we're about to free */
|
||||
if (bus_child_present(child) != 0) {
|
||||
pci_write_config(child, PCIR_COMMAND, pci_read_config(child,
|
||||
|
@ -31,6 +31,7 @@
|
||||
#define _PCIVAR_H_
|
||||
|
||||
#include <sys/queue.h>
|
||||
#include <sys/eventhandler.h>
|
||||
|
||||
/* some PCI bus constants */
|
||||
#define PCI_MAXMAPS_0 6 /* max. no. of memory/port maps */
|
||||
@ -631,4 +632,12 @@ void * vga_pci_map_bios(device_t dev, size_t *size);
|
||||
void vga_pci_unmap_bios(device_t dev, void *bios);
|
||||
int vga_pci_repost(device_t dev);
|
||||
|
||||
/**
|
||||
* Global eventhandlers invoked when PCI devices are added or removed
|
||||
* from the system.
|
||||
*/
|
||||
typedef void (*pci_event_fn)(void *arg, device_t dev);
|
||||
EVENTHANDLER_DECLARE(pci_add_device, pci_event_fn);
|
||||
EVENTHANDLER_DECLARE(pci_delete_device, pci_event_fn);
|
||||
|
||||
#endif /* _PCIVAR_H_ */
|
||||
|
Loading…
Reference in New Issue
Block a user