d/freebsd-skq
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the I/O MMU in bhyve when PCI devices are added and removed.

Browse Source 
When the I/O MMU is active in bhyve, all PCI devices need valid entries
in the DMAR context tables. The I/O MMU code does a single enumeration
of the available PCI devices during initialization to add all existing
devices to a domain representing the host. The ppt(4) driver then moves
pass through devices in and out of domains for virtual machines as needed.
However, when new PCI devices were added at runtime either via SR-IOV or
HotPlug, the I/O MMU tables were not updated.

This change adds a new set of EVENTHANDLERS that are invoked when PCI
devices are added and deleted. The I/O MMU driver in bhyve installs
handlers for these events which it uses to add and remove devices to
the "host" domain.

Reviewed by:	imp
Sponsored by:	Chelsio Communications
Differential Revision:	https://reviews.freebsd.org/D7667
This commit is contained in:
jhb 2016-09-06 20:17:54 +00:00
parent 87d2513314
commit 9b7bf59c96
4 changed files with 64 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
share/man/man9/pci.9
View File

@ -25,7 +25,7 @@
.\"
.\" $FreeBSD$
.\"
.Dd September 1, 2016
.Dd September 6, 2016
.Dt PCI 9
.Os
.Sh NAME
@ -149,6 +149,10 @@
.Fn pcie_read_config "device_t dev" "int reg" "int width"
.Ft void
.Fn pcie_write_config "device_t dev" "int reg" "uint32_t val" "int width"
.Ft void
.Fn pci_event_fn "void *arg" "device_t dev"
.Fn EVENTHANDLER_REGISTER "pci_add_device" "pci_event_fn"
.Fn EVENTHANDLER_DEREGISTER "pci_delete_resource" "pci_event_fn"
.In dev/pci/pci_iov.h
.Ft int
.Fn pci_iov_attach "device_t dev" "nvlist_t *pf_schema" "nvlist_t *vf_schema"
@ -910,6 +914,24 @@ with one in the new distribution.
The
.Fn pci_remap_msix
function will fail if this condition is not met.
.Ss Device Events
The
.Va pci_add_device
event handler is invoked every time a new PCI device is added to the system.
This includes the creation of Virtual Functions via SR-IOV.
.Pp
The
.Va pci_delete_device
event handler is invoked every time a PCI device is removed from the system.
.Pp
Both event handlers pass the
.Vt device_t
object of the relevant PCI device as
.Fa dev
to each callback function.
Both event handlers are invoked while
.Fa dev
is unattached but with valid instance variables.
.Sh SEE ALSO
.Xr pci 4 ,
.Xr pciconf 8 ,
@ -921,6 +943,7 @@ function will fail if this condition is not met.
.Xr devclass 9 ,
.Xr device 9 ,
.Xr driver 9 ,
.Xr eventhandler 9 ,
.Xr rman 9
.Rs
.%B FreeBSD Developers' Handbook

28
sys/amd64/vmm/io/iommu.c
View File

@ -58,6 +58,7 @@ SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0,
static struct iommu_ops *ops;
static void *host_domain;
static eventhandler_tag add_tag, delete_tag;
static __inline int
IOMMU_INIT(void)
@ -153,6 +154,21 @@ IOMMU_DISABLE(void)
(*ops->disable)();
}
static void
iommu_pci_add(void *arg, device_t dev)
{
/* Add new devices to the host domain. */
iommu_add_device(host_domain, pci_get_rid(dev));
}
static void
iommu_pci_delete(void *arg, device_t dev)
{
iommu_remove_device(host_domain, pci_get_rid(dev));
}
static void
iommu_init(void)
{
@ -195,6 +211,9 @@ iommu_init(void)
*/
iommu_create_mapping(host_domain, 0, 0, maxaddr);
add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0);
delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete,
NULL, 0);
for (bus = 0; bus <= PCI_BUSMAX; bus++) {
for (slot = 0; slot <= PCI_SLOTMAX; slot++) {
for (func = 0; func <= PCI_FUNCMAX; func++) {
@ -215,6 +234,15 @@ iommu_init(void)
void
iommu_cleanup(void)
{
if (add_tag != NULL) {
EVENTHANDLER_DEREGISTER(pci_add_device, add_tag);
add_tag = NULL;
}
if (delete_tag != NULL) {
EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag);
delete_tag = NULL;
}
IOMMU_DISABLE();
IOMMU_DESTROY_DOMAIN(host_domain);
IOMMU_CLEANUP();

3
sys/dev/pci/pci.c
View File

@ -4071,6 +4071,7 @@ pci_add_child(device_t bus, struct pci_devinfo *dinfo)
pci_print_verbose(dinfo);
pci_add_resources(bus, dinfo->cfg.dev, 0, 0);
pci_child_added(dinfo->cfg.dev);
EVENTHANDLER_INVOKE(pci_add_device, dinfo->cfg.dev);
}
void
@ -5312,6 +5313,8 @@ pci_child_deleted(device_t dev, device_t child)
dinfo = device_get_ivars(child);
rl = &dinfo->resources;
EVENTHANDLER_INVOKE(pci_delete_device, child);
/* Turn off access to resources we're about to free */
if (bus_child_present(child) != 0) {
pci_write_config(child, PCIR_COMMAND, pci_read_config(child,

9
sys/dev/pci/pcivar.h
View File

@ -31,6 +31,7 @@
#define _PCIVAR_H_
#include <sys/queue.h>
#include <sys/eventhandler.h>
/* some PCI bus constants */
#define PCI_MAXMAPS_0 6 /* max. no. of memory/port maps */
@ -631,4 +632,12 @@ void * vga_pci_map_bios(device_t dev, size_t *size);
void vga_pci_unmap_bios(device_t dev, void *bios);
int vga_pci_repost(device_t dev);
/**
* Global eventhandlers invoked when PCI devices are added or removed
* from the system.
*/
typedef void (*pci_event_fn)(void *arg, device_t dev);
EVENTHANDLER_DECLARE(pci_add_device, pci_event_fn);
EVENTHANDLER_DECLARE(pci_delete_device, pci_event_fn);
#endif /* _PCIVAR_H_ */