Add necessary bits for Linux KPI to work correctly on powerpc

PowerPC, and possibly other architectures, use different address ranges for
PCI space vs physical address space, which is only mapped at resource
activation time, when the BAR gets written.  The DRM kernel modules do not
activate the rman resources, soas not to waste KVA, instead only mapping
parts of the PCI memory at a time.  This introduces a
BUS_TRANSLATE_RESOURCE() method, implemented in the Open Firmware/FDT PCI
driver, to perform this necessary translation without activating the
resource.

In addition to system KPI changes, LinuxKPI is updated to handle a
big-endian host, by adding proper endian swaps to the I/O functions.

Submitted by:	mmacy
Reported by:	hselasky
Differential Revision:	https://reviews.freebsd.org/D21096

sys/compat/linuxkpi/common/include/linux/gfp.h

@@ -56,6 +56,7 @@
 
 #define	__GFP_IO	0
 #define	__GFP_NO_KSWAPD	0
+#define	__GFP_KSWAPD_RECLAIM	0
 #define	__GFP_WAIT	M_WAITOK
 #define	__GFP_DMA32	(1U << 24) /* LinuxKPI only */
 #define	__GFP_BITS_SHIFT 25

sys/compat/linuxkpi/common/include/linux/io.h

@@ -42,6 +42,32 @@
  * XXX This is all x86 specific.  It should be bus space access.
  */
 
+
+/* rmb and wmb are declared in machine/atomic.h, so should be included first. */
+#ifndef __io_br
+#define	__io_br()	__compiler_membar()
+#endif
+
+#ifndef __io_ar
+#ifdef rmb
+#define	__io_ar()	rmb()
+#else
+#define	__io_ar()	__compiler_membar()
+#endif
+#endif
+
+#ifndef __io_bw
+#ifdef wmb
+#define	__io_bw()	wmb()
+#else
+#define	__io_bw()	__compiler_membar()
+#endif
+#endif
+
+#ifndef __io_aw
+#define	__io_aw()	__compiler_membar()
+#endif
+
 /* Access MMIO registers atomically without barriers and byte swapping. */
 
 static inline uint8_t
@@ -112,9 +138,9 @@ readb(const volatile void *addr)
 {
 	uint8_t v;
 
-	__compiler_membar();
+	__io_br();
 	v = *(const volatile uint8_t *)addr;
-	__compiler_membar();
+	__io_ar();
 	return (v);
 }
 #define	readb(addr)		readb(addr)
@@ -123,9 +149,9 @@ readb(const volatile void *addr)
 static inline void
 writeb(uint8_t v, volatile void *addr)
 {
-	__compiler_membar();
+	__io_bw();
 	*(volatile uint8_t *)addr = v;
-	__compiler_membar();
+	__io_aw();
 }
 #define	writeb(v, addr)		writeb(v, addr)
 
@@ -135,9 +161,9 @@ readw(const volatile void *addr)
 {
 	uint16_t v;
 
-	__compiler_membar();
+	__io_br();
-	v = *(const volatile uint16_t *)addr;
+	v = le16toh(__raw_readw(addr));
-	__compiler_membar();
+	__io_ar();
 	return (v);
 }
 #define	readw(addr)		readw(addr)
@@ -146,9 +172,9 @@ readw(const volatile void *addr)
 static inline void
 writew(uint16_t v, volatile void *addr)
 {
-	__compiler_membar();
+	__io_bw();
-	*(volatile uint16_t *)addr = v;
+	__raw_writew(htole16(v), addr);
-	__compiler_membar();
+	__io_aw();
 }
 #define	writew(v, addr)		writew(v, addr)
 
@@ -158,9 +184,9 @@ readl(const volatile void *addr)
 {
 	uint32_t v;
 
-	__compiler_membar();
+	__io_br();
-	v = *(const volatile uint32_t *)addr;
+	v = le32toh(__raw_readl(addr));
-	__compiler_membar();
+	__io_ar();
 	return (v);
 }
 #define	readl(addr)		readl(addr)
@@ -169,9 +195,9 @@ readl(const volatile void *addr)
 static inline void
 writel(uint32_t v, volatile void *addr)
 {
-	__compiler_membar();
+	__io_bw();
-	*(volatile uint32_t *)addr = v;
+	__raw_writel(htole32(v), addr);
-	__compiler_membar();
+	__io_aw();
 }
 #define	writel(v, addr)		writel(v, addr)
 
@@ -183,9 +209,9 @@ readq(const volatile void *addr)
 {
 	uint64_t v;
 
-	__compiler_membar();
+	__io_br();
-	v = *(const volatile uint64_t *)addr;
+	v = le64toh(__raw_readq(addr));
-	__compiler_membar();
+	__io_ar();
 	return (v);
 }
 #define	readq(addr)		readq(addr)
@@ -193,9 +219,9 @@ readq(const volatile void *addr)
 static inline void
 writeq(uint64_t v, volatile void *addr)
 {
-	__compiler_membar();
+	__io_bw();
-	*(volatile uint64_t *)addr = v;
+	__raw_writeq(htole64(v), addr);
-	__compiler_membar();
+	__io_aw();
 }
 #define	writeq(v, addr)		writeq(v, addr)
 #endif
@@ -206,7 +232,7 @@ writeq(uint64_t v, volatile void *addr)
 static inline uint8_t
 readb_relaxed(const volatile void *addr)
 {
-	return (*(const volatile uint8_t *)addr);
+	return (__raw_readb(addr));
 }
 #define	readb_relaxed(addr)	readb_relaxed(addr)
 
@@ -214,7 +240,7 @@ readb_relaxed(const volatile void *addr)
 static inline void
 writeb_relaxed(uint8_t v, volatile void *addr)
 {
-	*(volatile uint8_t *)addr = v;
+	__raw_writeb(v, addr);
 }
 #define	writeb_relaxed(v, addr)	writeb_relaxed(v, addr)
 
@@ -222,7 +248,7 @@ writeb_relaxed(uint8_t v, volatile void *addr)
 static inline uint16_t
 readw_relaxed(const volatile void *addr)
 {
-	return (*(const volatile uint16_t *)addr);
+	return (le16toh(__raw_readw(addr)));
 }
 #define	readw_relaxed(addr)	readw_relaxed(addr)
 
@@ -230,7 +256,7 @@ readw_relaxed(const volatile void *addr)
 static inline void
 writew_relaxed(uint16_t v, volatile void *addr)
 {
-	*(volatile uint16_t *)addr = v;
+	__raw_writew(htole16(v), addr);
 }
 #define	writew_relaxed(v, addr)	writew_relaxed(v, addr)
 
@@ -238,7 +264,7 @@ writew_relaxed(uint16_t v, volatile void *addr)
 static inline uint32_t
 readl_relaxed(const volatile void *addr)
 {
-	return (*(const volatile uint32_t *)addr);
+	return (le32toh(__raw_readl(addr)));
 }
 #define	readl_relaxed(addr)	readl_relaxed(addr)
 
@@ -246,7 +272,7 @@ readl_relaxed(const volatile void *addr)
 static inline void
 writel_relaxed(uint32_t v, volatile void *addr)
 {
-	*(volatile uint32_t *)addr = v;
+	__raw_writel(htole32(v), addr);
 }
 #define	writel_relaxed(v, addr)	writel_relaxed(v, addr)
 
@@ -256,14 +282,14 @@ writel_relaxed(uint32_t v, volatile void *addr)
 static inline uint64_t
 readq_relaxed(const volatile void *addr)
 {
-	return (*(const volatile uint64_t *)addr);
+	return (le64toh(__raw_readq(addr)));
 }
 #define	readq_relaxed(addr)	readq_relaxed(addr)
 
 static inline void
 writeq_relaxed(uint64_t v, volatile void *addr)
 {
-	*(volatile uint64_t *)addr = v;
+	__raw_writeq(htole64(v), addr);
 }
 #define	writeq_relaxed(v, addr)	writeq_relaxed(v, addr)
 #endif
@@ -290,7 +316,13 @@ ioread16(const volatile void *addr)
 static inline uint16_t
 ioread16be(const volatile void *addr)
 {
-	return (bswap16(readw(addr)));
+	uint16_t v;
+
+	__io_br();
+	v = (be16toh(__raw_readw(addr)));
+	__io_ar();
+
+	return (v);
 }
 #define	ioread16be(addr)	ioread16be(addr)
 
@@ -306,7 +338,13 @@ ioread32(const volatile void *addr)
 static inline uint32_t
 ioread32be(const volatile void *addr)
 {
-	return (bswap32(readl(addr)));
+	uint32_t v;
+
+	__io_br();
+	v = (be32toh(__raw_readl(addr)));
+	__io_ar();
+
+	return (v);
 }
 #define	ioread32be(addr)	ioread32be(addr)
 
@@ -338,7 +376,9 @@ iowrite32(uint32_t v, volatile void *addr)
 static inline void
 iowrite32be(uint32_t v, volatile void *addr)
 {
-	writel(bswap32(v), addr);
+	__io_bw();
+	__raw_writel(htobe32(v), addr);
+	__io_aw();
 }
 #define	iowrite32be(v, addr)	iowrite32be(v, addr)
 

sys/compat/linuxkpi/common/include/linux/pci.h

@@ -205,6 +205,7 @@ struct pci_driver {
 
 struct pci_bus {
 	struct pci_dev	*self;
+	int		domain;
 	int		number;
 };
 
@@ -270,26 +271,6 @@ linux_pci_find_irq_dev(unsigned int irq)
 	return (found);
 }
 
-static inline unsigned long
-pci_resource_start(struct pci_dev *pdev, int bar)
-{
-	struct resource_list_entry *rle;
-
-	if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
-		return (0);
-	return rle->start;
-}
-
-static inline unsigned long
-pci_resource_len(struct pci_dev *pdev, int bar)
-{
-	struct resource_list_entry *rle;
-
-	if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
-		return (0);
-	return rle->count;
-}
-
 static inline int
 pci_resource_type(struct pci_dev *pdev, int bar)
 {
@@ -447,6 +428,9 @@ pci_disable_msix(struct pci_dev *pdev)
 	pdev->dev.msix_max = 0;
 }
 
+unsigned long	pci_resource_start(struct pci_dev *pdev, int bar);
+unsigned long	pci_resource_len(struct pci_dev *pdev, int bar);
+
 static inline bus_addr_t
 pci_bus_address(struct pci_dev *pdev, int bar)
 {
@@ -633,7 +617,7 @@ static inline void pci_disable_sriov(struct pci_dev *dev)
 /* XXX This should not be necessary. */
 #define	pcix_set_mmrbc(d, v)	0
 #define	pcix_get_max_mmrbc(d)	0
-#define	pcie_set_readrq(d, v)	0
+#define	pcie_set_readrq(d, v)	pci_set_max_read_req(&(d)->dev, (v))
 
 #define	PCI_DMA_BIDIRECTIONAL	0
 #define	PCI_DMA_TODEVICE	1

sys/compat/linuxkpi/common/src/linux_pci.c

@@ -29,12 +29,12 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/bus.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
 #include <sys/sysctl.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
-#include <sys/bus.h>
 #include <sys/fcntl.h>
 #include <sys/file.h>
 #include <sys/filio.h>
@@ -209,7 +209,6 @@ linux_pci_attach(device_t dev)
 	struct pci_driver *pdrv;
 	const struct pci_device_id *id;
 	device_t parent;
-	devclass_t devclass;
 	int error;
 
 	linux_set_current(curthread);
@@ -218,7 +217,6 @@ linux_pci_attach(device_t dev)
 	pdev = device_get_softc(dev);
 
 	parent = device_get_parent(dev);
-	devclass = device_get_devclass(parent);
 	if (pdrv->isdrm) {
 		dinfo = device_get_ivars(parent);
 		device_set_ivars(dev, dinfo);
@@ -254,6 +252,7 @@ linux_pci_attach(device_t dev)
 	pbus = malloc(sizeof(*pbus), M_DEVBUF, M_WAITOK | M_ZERO);
 	pbus->self = pdev;
 	pbus->number = pci_get_bus(dev);
+	pbus->domain = pci_get_domain(dev);
 	pdev->bus = pbus;
 
 	spin_lock(&pci_lock);
@@ -388,6 +387,36 @@ linux_pci_register_driver(struct pci_driver *pdrv)
 	return (_linux_pci_register_driver(pdrv, dc));
 }
 
+unsigned long
+pci_resource_start(struct pci_dev *pdev, int bar)
+{
+	struct resource_list_entry *rle;
+	unsigned long newstart;
+	device_t dev;
+
+	if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
+		return (0);
+	dev = pci_find_dbsf(pdev->bus->domain, pdev->bus->number,
+	    PCI_SLOT(pdev->devfn), PCI_FUNC(pdev->devfn));
+	MPASS(dev != NULL);
+	if (BUS_TRANSLATE_RESOURCE(dev, rle->type, rle->start, &newstart)) {
+		device_printf(pdev->dev.bsddev, "translate of %#lx failed\n",
+		    rle->start);
+		return (0);
+	}
+	return (newstart);
+}
+
+unsigned long
+pci_resource_len(struct pci_dev *pdev, int bar)
+{
+	struct resource_list_entry *rle;
+
+	if ((rle = linux_pci_get_bar(pdev, bar)) == NULL)
+		return (0);
+	return (rle->count);
+}
+
 int
 linux_pci_register_drm_driver(struct pci_driver *pdrv)
 {

sys/dev/ofw/ofwpci.c

@@ -76,6 +76,8 @@ static int ofw_pci_deactivate_resource(device_t, device_t, int, int,
     struct resource *);
 static int ofw_pci_adjust_resource(device_t, device_t, int,
     struct resource *, rman_res_t, rman_res_t);
+static int ofw_pci_translate_resource(device_t bus, int type,
+	rman_res_t start, rman_res_t *newstart);
 
 #ifdef __powerpc__
 static bus_space_tag_t ofw_pci_bus_get_bus_tag(device_t, device_t);
@@ -116,6 +118,7 @@ static device_method_t	ofw_pci_methods[] = {
 	DEVMETHOD(bus_activate_resource,	ofw_pci_activate_resource),
 	DEVMETHOD(bus_deactivate_resource,	ofw_pci_deactivate_resource),
 	DEVMETHOD(bus_adjust_resource,	ofw_pci_adjust_resource),
+	DEVMETHOD(bus_translate_resource,	ofw_pci_translate_resource),
 #ifdef __powerpc__
 	DEVMETHOD(bus_get_bus_tag,	ofw_pci_bus_get_bus_tag),
 #endif
@@ -478,6 +481,45 @@ ofw_pci_release_resource(device_t bus, device_t child, int type, int rid,
 	return (rman_release_resource(res));
 }
 
+static int
+ofw_pci_translate_resource(device_t bus, int type, rman_res_t start,
+	rman_res_t *newstart)
+{
+	struct ofw_pci_softc *sc;
+	struct ofw_pci_range *rp;
+	int space;
+
+	sc = device_get_softc(bus);
+
+	/*
+	 * Map this through the ranges list
+	 */
+	for (rp = sc->sc_range; rp < sc->sc_range + sc->sc_nrange &&
+	    rp->pci_hi != 0; rp++) {
+		if (start < rp->pci || start >= rp->pci + rp->size)
+			continue;
+
+		switch (rp->pci_hi & OFW_PCI_PHYS_HI_SPACEMASK) {
+		case OFW_PCI_PHYS_HI_SPACE_IO:
+			space = SYS_RES_IOPORT;
+			break;
+		case OFW_PCI_PHYS_HI_SPACE_MEM32:
+		case OFW_PCI_PHYS_HI_SPACE_MEM64:
+			space = SYS_RES_MEMORY;
+			break;
+		default:
+			space = -1;
+		}
+
+		if (type == space) {
+			start += (rp->host - rp->pci);
+			break;
+		}
+	}
+	*newstart = start;
+	return (0);
+}
+
 static int
 ofw_pci_activate_resource(device_t bus, device_t child, int type, int rid,
     struct resource *res)

sys/dev/pci/vga_pci.c

@@ -142,11 +142,35 @@ vga_pci_is_boot_display(device_t dev)
 	return (1);
 }
 
+static void
+vga_pci_reset(device_t dev)
+{
+	int ps;
+	/*
+	 * FLR is unsupported on GPUs so attempt a power-management reset by cycling
+	 * the device in/out of D3 state.
+	 * PCI spec says we can only go into D3 state from D0 state.
+	 * Transition from D[12] into D0 before going to D3 state.
+	 */
+	ps = pci_get_powerstate(dev);
+	if (ps != PCI_POWERSTATE_D0 && ps != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D0);
+	if (pci_get_powerstate(dev) != PCI_POWERSTATE_D3)
+		pci_set_powerstate(dev, PCI_POWERSTATE_D3);
+	pci_set_powerstate(dev, ps);
+}
+
+
 void *
 vga_pci_map_bios(device_t dev, size_t *size)
 {
-	int rid;
+	struct vga_resource *vr;
 	struct resource *res;
+	device_t pcib;
+	uint32_t rom_addr;
+	uint16_t config;
+	volatile char *bios;
+	int i, rid, found;
 
 #if defined(__amd64__) || defined(__i386__)
 	if (vga_pci_is_boot_display(dev)) {
@@ -164,21 +188,96 @@ vga_pci_map_bios(device_t dev, size_t *size)
 	}
 #endif
 
-	rid = PCIR_BIOS;
+	pcib = device_get_parent(device_get_parent(dev));
+	if (device_get_devclass(device_get_parent(pcib)) ==
+	    devclass_find("pci")) {
+		/*
+		 * The parent bridge is a PCI-to-PCI bridge: check the
+		 * value of the "VGA Enable" bit.
+		 */
+		config = pci_read_config(pcib, PCIR_BRIDGECTL_1, 2);
+		if ((config & PCIB_BCR_VGA_ENABLE) == 0) {
+			config |= PCIB_BCR_VGA_ENABLE;
+			pci_write_config(pcib, PCIR_BRIDGECTL_1, config, 2);
+		}
+	}
+
+	switch(pci_read_config(dev, PCIR_HDRTYPE, 1)) {
+	case PCIM_HDRTYPE_BRIDGE:
+		rid = PCIR_BIOS_1;
+		break;
+	case PCIM_HDRTYPE_CARDBUS:
+		rid = 0;
+		break;
+	default:
+		rid = PCIR_BIOS;
+		break;
+	}
+	if (rid == 0)
+		return (NULL);
+	res = vga_pci_alloc_resource(dev, NULL, SYS_RES_MEMORY, &rid, 0,
+	    ~0, 1, RF_ACTIVE);
+
+	if (res == NULL) {
+		device_printf(dev, "vga_pci_alloc_resource failed\n");
+		return (NULL);
+	}
+	bios = rman_get_virtual(res);
+	*size = rman_get_size(res);
+	for (found = i = 0; i < hz; i++) {
+		found = (bios[0] == 0x55 && bios[1] == 0xaa);
+		if (found)
+			break;
+		pause("vgabios", 1);
+	}
+	if (found)
+		return (__DEVOLATILE(void *, bios));
+	if (bootverbose)
+		device_printf(dev, "initial ROM mapping failed -- resetting\n");
+
+	/*
+	 * Enable ROM decode
+	 */
+	vga_pci_reset(dev);
+	rom_addr = pci_read_config(dev, rid, 4);
+	rom_addr &= 0x7ff;
+	rom_addr |= rman_get_start(res) | 0x1;
+	pci_write_config(dev, rid, rom_addr, 4);
+	vr = lookup_res(device_get_softc(dev), rid);
+	vga_pci_release_resource(dev, NULL, SYS_RES_MEMORY, rid,
+	    vr->vr_res);
+
+	/*
+	 * re-allocate
+	 */
 	res = vga_pci_alloc_resource(dev, NULL, SYS_RES_MEMORY, &rid, 0,
 	    ~0, 1, RF_ACTIVE);
 	if (res == NULL) {
+		device_printf(dev, "vga_pci_alloc_resource failed\n");
 		return (NULL);
 	}
-
+	bios = rman_get_virtual(res);
 	*size = rman_get_size(res);
-	return (rman_get_virtual(res));
+	for (found = i = 0; i < 3*hz; i++) {
+		found = (bios[0] == 0x55 && bios[1] == 0xaa);
+		if (found)
+			break;
+		pause("vgabios", 1);
+	}
+	if (found)
+		return (__DEVOLATILE(void *, bios));
+	device_printf(dev, "ROM mapping failed\n");
+	vr = lookup_res(device_get_softc(dev), rid);
+	vga_pci_release_resource(dev, NULL, SYS_RES_MEMORY, rid,
+	    vr->vr_res);
+	return (NULL);
 }
 
 void
 vga_pci_unmap_bios(device_t dev, void *bios)
 {
 	struct vga_resource *vr;
+	int rid;
 
 	if (bios == NULL) {
 		return;
@@ -192,16 +291,28 @@ vga_pci_unmap_bios(device_t dev, void *bios)
 		return;
 	}
 #endif
-
+	switch(pci_read_config(dev, PCIR_HDRTYPE, 1)) {
+	case PCIM_HDRTYPE_BRIDGE:
+		rid = PCIR_BIOS_1;
+		break;
+	case PCIM_HDRTYPE_CARDBUS:
+		rid = 0;
+		break;
+	default:
+		rid = PCIR_BIOS;
+		break;
+	}
+	if (rid == 0)
+		return;
 	/*
 	 * Look up the PCIR_BIOS resource in our softc.  It should match
 	 * the address we returned previously.
 	 */
-	vr = lookup_res(device_get_softc(dev), PCIR_BIOS);
+	vr = lookup_res(device_get_softc(dev), rid);
 	KASSERT(vr->vr_res != NULL, ("vga_pci_unmap_bios: bios not mapped"));
 	KASSERT(rman_get_virtual(vr->vr_res) == bios,
 	    ("vga_pci_unmap_bios: mismatch"));
-	vga_pci_release_resource(dev, NULL, SYS_RES_MEMORY, PCIR_BIOS,
+	vga_pci_release_resource(dev, NULL, SYS_RES_MEMORY, rid,
 	    vr->vr_res);
 }
 

sys/kern/bus_if.m

@@ -76,6 +76,18 @@ CODE {
 	{
 		return (0);
 	}
+
+	static int
+	null_translate_resource(device_t bus, int type, rman_res_t start,
+		rman_res_t *newstart)
+	{
+		if (device_get_parent(bus) != NULL)
+			return (BUS_TRANSLATE_RESOURCE(device_get_parent(bus),
+			    type, start, newstart));
+
+		*newstart = start;
+		return (0);
+	}
 };
 
 /**
@@ -406,6 +418,23 @@ METHOD int adjust_resource {
 	rman_res_t	_end;
 };
 
+
+/**
+ * @brief translate a resource value
+ *
+ *
+ * @param _dev		the device associated with the resource
+ * @param _type		the type of resource
+ * @param _start	the starting address of the resource range
+ * @param _newstart	the new starting address of the resource range
+ */
+METHOD int translate_resource {
+	device_t	_dev;
+	int		_type;
+	rman_res_t	_start;
+	rman_res_t	*_newstart;
+} DEFAULT null_translate_resource;
+
 /**
  * @brief Release a resource
  *

sys/kern/subr_bus.c

@@ -3971,7 +3971,6 @@ bus_helper_reset_prepare(device_t dev, int flags)
 	return (0);
 }
 
-
 /**
  * @brief Helper function for implementing BUS_PRINT_CHILD().
  *

sys/sys/bus.h

@@ -434,6 +434,8 @@ struct resource *
 	bus_generic_alloc_resource(device_t bus, device_t child, int type,
 				   int *rid, rman_res_t start, rman_res_t end,
 				   rman_res_t count, u_int flags);
+int	bus_generic_translate_resource(device_t dev, int type, rman_res_t start,
+			      rman_res_t *newstart);
 int	bus_generic_attach(device_t dev);
 int	bus_generic_bind_intr(device_t dev, device_t child,
 			      struct resource *irq, int cpu);