Add a bus method to fetch the VM domain for the given device/bus.

* Add a bus_if.m method - get_domain() - returning the VM domain or
  ENOENT if the device isn't in a VM domain;
* Add bus methods to print out the domain of the device if appropriate;
* Add code in srat.c to save the PXM -> VM domain mapping that's done and
  expose a function to translate VM domain -> PXM;
* Add ACPI and ACPI PCI methods to check if the bus has a _PXM attribute
  and if so map it to the VM domain;
* (.. yes, this works recursively.)
* Have the pci bus glue print out the device VM domain if present.

Note: this is just the plumbing to start enumerating information -
it doesn't at all modify behaviour.

Differential Revision:	D906
Reviewed by:	jhb
Sponsored by:	Norse Corp

sys/dev/acpica/acpi.c

@@ -208,6 +208,7 @@ static device_method_t acpi_methods[] = {
     DEVMETHOD(bus_setup_intr,		bus_generic_setup_intr),
     DEVMETHOD(bus_teardown_intr,	bus_generic_teardown_intr),
     DEVMETHOD(bus_hint_device_unit,	acpi_hint_device_unit),
+    DEVMETHOD(bus_get_domain,		acpi_get_domain),
 
     /* ACPI bus */
     DEVMETHOD(acpi_id_probe,		acpi_device_id_probe),
@@ -794,6 +795,7 @@ acpi_print_child(device_t bus, device_t child)
     retval += resource_list_print_type(rl, "drq",   SYS_RES_DRQ,    "%ld");
     if (device_get_flags(child))
 	retval += printf(" flags %#x", device_get_flags(child));
+    retval += bus_print_child_domain(bus, child);
     retval += bus_print_child_footer(bus, child);
 
     return (retval);
@@ -1066,6 +1068,35 @@ acpi_hint_device_unit(device_t acdev, device_t child, const char *name,
     }
 }
 
+/*
+ * Fech the NUMA domain for the given device.
+ *
+ * If a device has a _PXM method, map that to a NUMA domain.
+ *
+ * If none is found, then it'll call the parent method.
+ * If there's no domain, return ENOENT.
+ */
+int
+acpi_get_domain(device_t dev, device_t child, int *domain)
+{
+#if MAXMEMDOM > 1
+	ACPI_HANDLE h;
+	int d, pxm;
+
+	h = acpi_get_handle(child);
+	if ((h != NULL) &&
+	    ACPI_SUCCESS(acpi_GetInteger(h, "_PXM", &pxm))) {
+		d = acpi_map_pxm_to_vm_domainid(pxm);
+		if (d < 0)
+			return (ENOENT);
+		*domain = d;
+		return (0);
+	}
+#endif
+	/* No _PXM node; go up a level */
+	return (bus_generic_get_domain(dev, child, domain));
+}
+
 /*
  * Pre-allocate/manage all memory and IO resources.  Since rman can't handle
  * duplicates, we merge any in the sysresource attach routine.

sys/dev/acpica/acpi_pci.c

@@ -94,6 +94,7 @@ static device_method_t acpi_pci_methods[] = {
 	DEVMETHOD(bus_write_ivar,	acpi_pci_write_ivar),
 	DEVMETHOD(bus_child_location_str, acpi_pci_child_location_str_method),
 	DEVMETHOD(bus_get_dma_tag,	acpi_pci_get_dma_tag),
+	DEVMETHOD(bus_get_domain,	acpi_get_domain),
 
 	/* PCI interface */
 	DEVMETHOD(pci_set_powerstate,	acpi_pci_set_powerstate_method),

sys/dev/acpica/acpivar.h

@@ -489,5 +489,16 @@ ACPI_HANDLE	acpi_GetReference(ACPI_HANDLE scope, ACPI_OBJECT *obj);
 
 SYSCTL_DECL(_debug_acpi);
 
+/*
+ * Map a PXM to a VM domain.
+ *
+ * Returns the VM domain ID if found, or -1 if not found / invalid.
+ */
+#if MAXMEMDOM > 1
+extern	int acpi_map_pxm_to_vm_domainid(int pxm);
+#endif
+
+extern	int acpi_get_domain(device_t dev, device_t child, int *domain);
+
 #endif /* _KERNEL */
 #endif /* !_ACPIVAR_H_ */

sys/dev/pci/pci.c

@@ -3965,6 +3965,7 @@ pci_print_child(device_t dev, device_t child)
 	retval += printf(" at device %d.%d", pci_get_slot(child),
 	    pci_get_function(child));
 
+	retval += bus_print_child_domain(dev, child);
 	retval += bus_print_child_footer(dev, child);
 
 	return (retval);

sys/kern/bus_if.m

@@ -692,3 +692,16 @@ METHOD int resume_child {
 	device_t	_dev;
 	device_t	_child;
 } DEFAULT bus_generic_resume_child;
+
+/**
+ * @brief Get the VM domain handle for the given bus and child.
+ *
+ * @param _dev		the bus device
+ * @param _child	the child device
+ * @param _domain	a pointer to the bus's domain handle identifier
+ */
+METHOD int get_domain {
+	device_t	_dev;
+	device_t	_child;
+	int		*_domain;
+} DEFAULT bus_generic_get_domain;

sys/kern/subr_bus.c

@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/uio.h>
 #include <sys/bus.h>
 #include <sys/interrupt.h>
+#include <sys/cpuset.h>
 
 #include <net/vnet.h>
 
@@ -3751,6 +3752,25 @@ bus_print_child_footer(device_t dev, device_t child)
 	return (printf(" on %s\n", device_get_nameunit(dev)));
 }
 
+/**
+ * @brief Helper function for implementing BUS_PRINT_CHILD().
+ *
+ * This function prints out the VM domain for the given device.
+ *
+ * @returns the number of characters printed
+ */
+int
+bus_print_child_domain(device_t dev, device_t child)
+{
+	int domain;
+
+	/* No domain? Don't print anything */
+	if (BUS_GET_DOMAIN(dev, child, &domain) != 0)
+		return (0);
+
+	return (printf(" numa-domain %d", domain));
+}
+
 /**
  * @brief Helper function for implementing BUS_PRINT_CHILD().
  *
@@ -3765,6 +3785,7 @@ bus_generic_print_child(device_t dev, device_t child)
 	int	retval = 0;
 
 	retval += bus_print_child_header(dev, child);
+	retval += bus_print_child_domain(dev, child);
 	retval += bus_print_child_footer(dev, child);
 
 	return (retval);
@@ -4179,6 +4200,16 @@ bus_generic_child_present(device_t dev, device_t child)
 	return (BUS_CHILD_PRESENT(device_get_parent(dev), dev));
 }
 
+int
+bus_generic_get_domain(device_t dev, device_t child, int *domain)
+{
+
+	if (dev->parent)
+		return (BUS_GET_DOMAIN(dev->parent, dev, domain));
+
+	return (ENOENT);
+}
+
 /*
  * Some convenience functions to make it easier for drivers to use the
  * resource-management functions.  All these really do is hide the

sys/sys/bus.h

@@ -331,6 +331,7 @@ struct resource_list *
 	bus_generic_get_resource_list (device_t, device_t);
 void	bus_generic_new_pass(device_t dev);
 int	bus_print_child_header(device_t dev, device_t child);
+int	bus_print_child_domain(device_t dev, device_t child);
 int	bus_print_child_footer(device_t dev, device_t child);
 int	bus_generic_print_child(device_t dev, device_t child);
 int	bus_generic_probe(device_t dev);
@@ -364,6 +365,8 @@ int	bus_generic_teardown_intr(device_t dev, device_t child,
 int	bus_generic_write_ivar(device_t dev, device_t child, int which,
 			       uintptr_t value);
 
+int	bus_generic_get_domain(device_t dev, device_t child, int *domain);
+
 /*
  * Wrapper functions for the BUS_*_RESOURCE methods to make client code
  * a little simpler.