ntb: Add Intel Xeon Gen3 support
The NTB hardware starting with Skylake has some changes to the register map and the doorbell interface. Add a new NTB_XEON_GEN3 device type and use it to conditionalize driver logic that differs from the existing Xeon code. Reviewed by: vangyzen Discussed with: cem, Bret Ketchum <Bret.Ketchum@dell.com> MFC after: 1 month Sponsored by: NetApp, Inc. Sponsored by: Klara, Inc. Differential Revision: https://reviews.freebsd.org/D26683
This commit is contained in:
Mark Johnston
parent
97441fab87
commit
6660ef6e91
@ -25,7 +25,7 @@
|
||||
.\"
|
||||
.\" $FreeBSD$
|
||||
.\"
|
||||
.Dd August 30, 2017
|
||||
.Dd October 11, 2020
|
||||
.Dt NTB_HW_INTEL 4
|
||||
.Os
|
||||
.Sh NAME
|
||||
@ -50,16 +50,18 @@ The
|
||||
driver provides support for the Non-Transparent Bridge (NTB) hardware in
|
||||
Intel Xeon E3/E5 and S1200 processor families, which allow one of their PCIe
|
||||
ports to be switched from transparent to non-transparent bridge mode.
|
||||
In this mode bridge looks not as a PCI bridge, but as PCI endpoint device.
|
||||
In this mode the bridge looks not like a PCI bridge, but like a PCI endpoint
|
||||
device.
|
||||
The driver hides hardware details, exposing memory windows, scratchpads
|
||||
and doorbells of the other side via hardware independent KPI to
|
||||
and doorbells of the other side via a hardware independent KPI to the
|
||||
.Xr ntb 4
|
||||
subsystem.
|
||||
.Pp
|
||||
The hardware provides 2 or 3 memory windows to the other system's memory,
|
||||
16 scratchpad registers and 14 or 34 doorbells to interrupt the other system.
|
||||
On Xeon processors one of memory windows is typically consumed by the driver
|
||||
itself to workaround multiple hardware erratas.
|
||||
16 scratchpad registers and 14, 31 or 34 doorbells to interrupt the other
|
||||
system, depending on the platform.
|
||||
On Xeon processors one of the memory windows is typically consumed by the driver
|
||||
itself to work around multiple hardware errata.
|
||||
.Sh CONFIGURATION
|
||||
The NTB configuration should be set by BIOS.
|
||||
It includes enabling NTB, choosing between NTB-to-NTB (back-to-back) or
|
||||
@ -67,9 +69,10 @@ NTB-to-Root Port mode,
|
||||
enabling split BAR mode (one of two 64-bit BARs can be split into two 32-bit
|
||||
ones) and configuring BAR sizes in bits (from 12 to 29/39) for both NTB sides.
|
||||
.Pp
|
||||
The recommended configuration is NTB-to-NTB mode, split bar is enabled and
|
||||
all BAR sizes are set to 20 (1 MiB).
|
||||
The recommended configuration is NTB-to-NTB mode, split bar enabled and
|
||||
all BAR sizes set to 20 (1 MiB).
|
||||
This needs to be done on both systems.
|
||||
Note, on Xeon SkyLake and newer platforms, split bar mode is not available.
|
||||
.Sh SEE ALSO
|
||||
.Xr if_ntb 4 ,
|
||||
.Xr ntb_transport 4 ,
|
||||
|
||||
@ -68,7 +68,8 @@ __FBSDID("$FreeBSD$");
|
||||
#include "ntb_hw_intel.h"
|
||||
#include "../ntb.h"
|
||||
|
||||
#define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT)
|
||||
#define MAX_MSIX_INTERRUPTS \
|
||||
MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT)
|
||||
|
||||
#define NTB_HB_TIMEOUT 1 /* second */
|
||||
#define ATOM_LINK_RECOVERY_TIME 500 /* ms */
|
||||
@ -87,7 +88,8 @@ __FBSDID("$FreeBSD$");
|
||||
#define PCI_MSIX_ENTRY_DATA 8
|
||||
|
||||
enum ntb_device_type {
|
||||
NTB_XEON,
|
||||
NTB_XEON_GEN1,
|
||||
NTB_XEON_GEN3,
|
||||
NTB_ATOM
|
||||
};
|
||||
|
||||
@ -338,6 +340,7 @@ static int map_memory_window_bar(struct ntb_softc *ntb,
|
||||
static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb);
|
||||
static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail);
|
||||
static int intel_ntb_init_isr(struct ntb_softc *ntb);
|
||||
static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb);
|
||||
static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb);
|
||||
static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors);
|
||||
static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb);
|
||||
@ -355,8 +358,10 @@ static void intel_ntb_exchange_msix(void *);
|
||||
static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id);
|
||||
static void intel_ntb_detect_max_mw(struct ntb_softc *ntb);
|
||||
static int intel_ntb_detect_xeon(struct ntb_softc *ntb);
|
||||
static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb);
|
||||
static int intel_ntb_detect_atom(struct ntb_softc *ntb);
|
||||
static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb);
|
||||
static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb);
|
||||
static int intel_ntb_atom_init_dev(struct ntb_softc *ntb);
|
||||
static void intel_ntb_teardown_xeon(struct ntb_softc *ntb);
|
||||
static void configure_atom_secondary_side_bars(struct ntb_softc *ntb);
|
||||
@ -368,6 +373,9 @@ static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr,
|
||||
enum ntb_bar idx);
|
||||
static int xeon_setup_b2b_mw(struct ntb_softc *,
|
||||
const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr);
|
||||
static int xeon_gen3_setup_b2b_mw(struct ntb_softc *);
|
||||
static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr,
|
||||
size_t size);
|
||||
static inline bool link_is_up(struct ntb_softc *ntb);
|
||||
static inline bool _xeon_link_is_up(struct ntb_softc *ntb);
|
||||
static inline bool atom_link_is_err(struct ntb_softc *ntb);
|
||||
@ -479,6 +487,7 @@ SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx,
|
||||
#define NTB_B2BDOORBELL_BIT14 (1 << 3)
|
||||
/* Software/configuration owns the top 16 bits. */
|
||||
#define NTB_SPLIT_BAR (1ull << 16)
|
||||
#define NTB_ONE_MSIX (1ull << 17)
|
||||
|
||||
#define NTB_FEATURES_STR \
|
||||
"\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \
|
||||
@ -490,18 +499,21 @@ static struct ntb_hw_info pci_ids[] = {
|
||||
NTB_ATOM, 0 },
|
||||
|
||||
{ 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B",
|
||||
NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
|
||||
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
|
||||
{ 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B",
|
||||
NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
|
||||
{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON,
|
||||
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 },
|
||||
{ 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B",
|
||||
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K },
|
||||
{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON,
|
||||
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
{ 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B",
|
||||
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
NTB_SB01BASE_LOCKUP },
|
||||
{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON,
|
||||
NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
{ 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B",
|
||||
NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 |
|
||||
NTB_SB01BASE_LOCKUP },
|
||||
|
||||
{ 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B",
|
||||
NTB_XEON_GEN3, 0 },
|
||||
};
|
||||
|
||||
static const struct ntb_reg atom_reg = {
|
||||
@ -586,6 +598,37 @@ static struct ntb_b2b_addr xeon_b2b_dsd_addr = {
|
||||
.bar5_addr32 = XEON_B2B_BAR5_ADDR32,
|
||||
};
|
||||
|
||||
static const struct ntb_reg xeon_gen3_reg = {
|
||||
.ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL,
|
||||
.lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET,
|
||||
.db_size = sizeof(uint32_t),
|
||||
.mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 },
|
||||
};
|
||||
|
||||
static const struct ntb_alt_reg xeon_gen3_pri_reg = {
|
||||
.db_bell = XEON_GEN3_REG_EMDOORBELL,
|
||||
.db_mask = XEON_GEN3_REG_IMINT_DISABLE,
|
||||
.spad = XEON_GEN3_REG_IMSPAD,
|
||||
};
|
||||
|
||||
static const struct ntb_alt_reg xeon_gen3_b2b_reg = {
|
||||
.db_bell = XEON_GEN3_REG_IMDOORBELL,
|
||||
.db_mask = XEON_GEN3_REG_EMINT_DISABLE,
|
||||
.spad = XEON_GEN3_REG_IMB2B_SSPAD,
|
||||
};
|
||||
|
||||
static const struct ntb_xlat_reg xeon_gen3_sec_xlat = {
|
||||
.bar0_base = XEON_GEN3_EXT_REG_BAR0BASE,
|
||||
.bar2_base = XEON_GEN3_EXT_REG_BAR1BASE,
|
||||
.bar4_base = XEON_GEN3_EXT_REG_BAR2BASE,
|
||||
|
||||
.bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT,
|
||||
.bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT,
|
||||
|
||||
.bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE,
|
||||
.bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE,
|
||||
};
|
||||
|
||||
SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
|
||||
"B2B MW segment overrides -- MUST be the same on both sides");
|
||||
|
||||
@ -662,6 +705,8 @@ intel_ntb_attach(device_t device)
|
||||
|
||||
if (ntb->type == NTB_ATOM)
|
||||
error = intel_ntb_detect_atom(ntb);
|
||||
else if (ntb->type == NTB_XEON_GEN3)
|
||||
error = intel_ntb_detect_xeon_gen3(ntb);
|
||||
else
|
||||
error = intel_ntb_detect_xeon(ntb);
|
||||
if (error != 0)
|
||||
@ -676,6 +721,8 @@ intel_ntb_attach(device_t device)
|
||||
goto out;
|
||||
if (ntb->type == NTB_ATOM)
|
||||
error = intel_ntb_atom_init_dev(ntb);
|
||||
else if (ntb->type == NTB_XEON_GEN3)
|
||||
error = intel_ntb_xeon_gen3_init_dev(ntb);
|
||||
else
|
||||
error = intel_ntb_xeon_init_dev(ntb);
|
||||
if (error != 0)
|
||||
@ -715,7 +762,7 @@ intel_ntb_detach(device_t device)
|
||||
callout_drain(&ntb->lr_timer);
|
||||
callout_drain(&ntb->peer_msix_work);
|
||||
pci_disable_busmaster(ntb->device);
|
||||
if (ntb->type == NTB_XEON)
|
||||
if (ntb->type == NTB_XEON_GEN1)
|
||||
intel_ntb_teardown_xeon(ntb);
|
||||
intel_ntb_teardown_interrupts(ntb);
|
||||
|
||||
@ -825,22 +872,39 @@ intel_ntb_map_pci_bars(struct ntb_softc *ntb)
|
||||
rc = map_memory_window_bar(ntb, bar);
|
||||
if (rc != 0)
|
||||
goto out;
|
||||
bar->psz_off = XEON_PBAR23SZ_OFFSET;
|
||||
bar->ssz_off = XEON_SBAR23SZ_OFFSET;
|
||||
bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
|
||||
if (ntb->type == NTB_XEON_GEN3) {
|
||||
bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ;
|
||||
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ;
|
||||
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE;
|
||||
} else {
|
||||
bar->psz_off = XEON_PBAR23SZ_OFFSET;
|
||||
bar->ssz_off = XEON_SBAR23SZ_OFFSET;
|
||||
bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET;
|
||||
}
|
||||
|
||||
bar = &ntb->bar_info[NTB_B2B_BAR_2];
|
||||
bar->pci_resource_id = PCIR_BAR(4);
|
||||
rc = map_memory_window_bar(ntb, bar);
|
||||
if (rc != 0)
|
||||
goto out;
|
||||
bar->psz_off = XEON_PBAR4SZ_OFFSET;
|
||||
bar->ssz_off = XEON_SBAR4SZ_OFFSET;
|
||||
bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
|
||||
if (ntb->type == NTB_XEON_GEN3) {
|
||||
bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ;
|
||||
bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ;
|
||||
bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE;
|
||||
} else {
|
||||
bar->psz_off = XEON_PBAR4SZ_OFFSET;
|
||||
bar->ssz_off = XEON_SBAR4SZ_OFFSET;
|
||||
bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET;
|
||||
}
|
||||
|
||||
if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR))
|
||||
goto out;
|
||||
|
||||
if (ntb->type == NTB_XEON_GEN3) {
|
||||
device_printf(ntb->device, "no split bar support\n");
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
bar = &ntb->bar_info[NTB_B2B_BAR_3];
|
||||
bar->pci_resource_id = PCIR_BAR(5);
|
||||
rc = map_memory_window_bar(ntb, bar);
|
||||
@ -1059,6 +1123,63 @@ intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail)
|
||||
return (rc);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb)
|
||||
{
|
||||
uint64_t i, reg;
|
||||
uint32_t desired_vectors, num_vectors;
|
||||
int rc;
|
||||
|
||||
ntb->allocated_interrupts = 0;
|
||||
ntb->last_ts = ticks;
|
||||
|
||||
/* Mask all the interrupts, including hardware interrupt */
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL);
|
||||
|
||||
/* Clear Interrupt Status */
|
||||
reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS);
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg);
|
||||
|
||||
num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device),
|
||||
XEON_GEN3_DB_MSIX_VECTOR_COUNT);
|
||||
|
||||
rc = pci_alloc_msix(ntb->device, &num_vectors);
|
||||
if (rc != 0) {
|
||||
device_printf(ntb->device,
|
||||
"Interrupt allocation failed %d\n", rc);
|
||||
return (rc);
|
||||
}
|
||||
if (desired_vectors != num_vectors) {
|
||||
device_printf(ntb->device, "Couldn't get %d vectors\n",
|
||||
XEON_GEN3_DB_MSIX_VECTOR_COUNT);
|
||||
return (ENXIO);
|
||||
}
|
||||
/* 32 db + 1 hardware */
|
||||
if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) {
|
||||
/* Program INTVECXX source register */
|
||||
for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) {
|
||||
/* interrupt source i for vector i */
|
||||
intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i);
|
||||
if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) {
|
||||
intel_ntb_reg_write(1,
|
||||
XEON_GEN3_REG_IMINTVEC00 + i,
|
||||
XEON_GEN3_LINK_VECTOR_INDEX);
|
||||
}
|
||||
}
|
||||
|
||||
intel_ntb_create_msix_vec(ntb, num_vectors);
|
||||
rc = intel_ntb_setup_msix(ntb, num_vectors);
|
||||
|
||||
/* enable all interrupts */
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL);
|
||||
} else {
|
||||
device_printf(ntb->device, "need to remap interrupts, giving up.\n");
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_init_isr(struct ntb_softc *ntb)
|
||||
{
|
||||
@ -1098,7 +1219,7 @@ intel_ntb_init_isr(struct ntb_softc *ntb)
|
||||
} else
|
||||
num_vectors = 1;
|
||||
|
||||
if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) {
|
||||
if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) {
|
||||
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
|
||||
device_printf(ntb->device,
|
||||
"Errata workaround does not support MSI or INTX\n");
|
||||
@ -1177,20 +1298,17 @@ intel_ntb_teardown_interrupts(struct ntb_softc *ntb)
|
||||
pci_release_msi(ntb->device);
|
||||
}
|
||||
|
||||
/*
|
||||
* Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it
|
||||
* out to make code clearer.
|
||||
*/
|
||||
static inline uint64_t
|
||||
db_ioread(struct ntb_softc *ntb, uint64_t regoff)
|
||||
{
|
||||
|
||||
if (ntb->type == NTB_ATOM)
|
||||
switch (ntb->type) {
|
||||
case NTB_ATOM:
|
||||
case NTB_XEON_GEN3:
|
||||
return (intel_ntb_reg_read(8, regoff));
|
||||
|
||||
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
|
||||
|
||||
return (intel_ntb_reg_read(2, regoff));
|
||||
case NTB_XEON_GEN1:
|
||||
return (intel_ntb_reg_read(2, regoff));
|
||||
}
|
||||
}
|
||||
|
||||
static inline void
|
||||
@ -1211,13 +1329,15 @@ static inline void
|
||||
db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val)
|
||||
{
|
||||
|
||||
if (ntb->type == NTB_ATOM) {
|
||||
switch (ntb->type) {
|
||||
case NTB_ATOM:
|
||||
case NTB_XEON_GEN3:
|
||||
intel_ntb_reg_write(8, regoff, val);
|
||||
return;
|
||||
break;
|
||||
case NTB_XEON_GEN1:
|
||||
intel_ntb_reg_write(2, regoff, (uint16_t)val);
|
||||
break;
|
||||
}
|
||||
|
||||
KASSERT(ntb->type == NTB_XEON, ("bad ntb type"));
|
||||
intel_ntb_reg_write(2, regoff, (uint16_t)val);
|
||||
}
|
||||
|
||||
static void
|
||||
@ -1267,8 +1387,10 @@ intel_ntb_db_read(device_t dev)
|
||||
|
||||
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))
|
||||
return (ntb->fake_db);
|
||||
|
||||
return (db_ioread(ntb, ntb->self_reg->db_bell));
|
||||
if (ntb->type == NTB_XEON_GEN3)
|
||||
return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
|
||||
else
|
||||
return (db_ioread(ntb, ntb->self_reg->db_bell));
|
||||
}
|
||||
|
||||
static void
|
||||
@ -1288,7 +1410,11 @@ intel_ntb_db_clear(device_t dev, uint64_t bits)
|
||||
return;
|
||||
}
|
||||
|
||||
db_iowrite(ntb, ntb->self_reg->db_bell, bits);
|
||||
if (ntb->type == NTB_XEON_GEN3)
|
||||
intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS,
|
||||
(uint32_t)bits);
|
||||
else
|
||||
db_iowrite(ntb, ntb->self_reg->db_bell, bits);
|
||||
}
|
||||
|
||||
static inline uint64_t
|
||||
@ -1322,9 +1448,14 @@ intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec)
|
||||
ntb->last_ts = ticks;
|
||||
vec_mask = intel_ntb_vec_mask(ntb, vec);
|
||||
|
||||
if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX)
|
||||
vec_mask |= ntb->db_link_mask;
|
||||
if ((vec_mask & ntb->db_link_mask) != 0) {
|
||||
if (intel_ntb_poll_link(ntb))
|
||||
ntb_link_event(ntb->device);
|
||||
if (ntb->type == NTB_XEON_GEN3)
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS,
|
||||
intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS));
|
||||
}
|
||||
|
||||
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) &&
|
||||
@ -1449,15 +1580,23 @@ static void
|
||||
intel_ntb_detect_max_mw(struct ntb_softc *ntb)
|
||||
{
|
||||
|
||||
if (ntb->type == NTB_ATOM) {
|
||||
switch (ntb->type) {
|
||||
case NTB_ATOM:
|
||||
ntb->mw_count = ATOM_MW_COUNT;
|
||||
return;
|
||||
break;
|
||||
case NTB_XEON_GEN1:
|
||||
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
|
||||
ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
|
||||
else
|
||||
ntb->mw_count = XEON_SNB_MW_COUNT;
|
||||
break;
|
||||
case NTB_XEON_GEN3:
|
||||
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
|
||||
ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT;
|
||||
else
|
||||
ntb->mw_count = XEON_GEN3_MW_COUNT;
|
||||
break;
|
||||
}
|
||||
|
||||
if (HAS_FEATURE(ntb, NTB_SPLIT_BAR))
|
||||
ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT;
|
||||
else
|
||||
ntb->mw_count = XEON_SNB_MW_COUNT;
|
||||
}
|
||||
|
||||
static int
|
||||
@ -1533,6 +1672,54 @@ intel_ntb_detect_atom(struct ntb_softc *ntb)
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb)
|
||||
{
|
||||
uint8_t ppd, conn_type;
|
||||
|
||||
ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1);
|
||||
ntb->ppd = ppd;
|
||||
|
||||
/* check port definition */
|
||||
conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd);
|
||||
switch (conn_type) {
|
||||
case NTB_CONN_B2B:
|
||||
ntb->conn_type = conn_type;
|
||||
break;
|
||||
default:
|
||||
device_printf(ntb->device, "Unsupported connection type: %u\n",
|
||||
conn_type);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
/* check cross link configuration status */
|
||||
if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) {
|
||||
/* NTB Port is configured as DSD/USP */
|
||||
ntb->dev_type = NTB_DEV_DSD;
|
||||
} else {
|
||||
/* NTB Port is configured as USD/DSP */
|
||||
ntb->dev_type = NTB_DEV_USD;
|
||||
}
|
||||
|
||||
if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) {
|
||||
/*
|
||||
* This bit when set, causes only a single MSI-X message to be
|
||||
* generated if MSI-X is enabled.
|
||||
*/
|
||||
ntb->features |= NTB_ONE_MSIX;
|
||||
}
|
||||
|
||||
if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) {
|
||||
/* BARs 4 and 5 are presented as two 32b non-prefetchable BARs */
|
||||
ntb->features |= NTB_SPLIT_BAR;
|
||||
}
|
||||
|
||||
device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x,"
|
||||
"features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
|
||||
{
|
||||
@ -1617,6 +1804,42 @@ intel_ntb_xeon_init_dev(struct ntb_softc *ntb)
|
||||
return (rc);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb)
|
||||
{
|
||||
int rc;
|
||||
|
||||
ntb->spad_count = XEON_GEN3_SPAD_COUNT;
|
||||
ntb->db_count = XEON_GEN3_DB_COUNT;
|
||||
ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT;
|
||||
ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT;
|
||||
ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT;
|
||||
|
||||
if (ntb->conn_type != NTB_CONN_B2B) {
|
||||
device_printf(ntb->device, "Connection type %d not supported\n",
|
||||
ntb->conn_type);
|
||||
return (ENXIO);
|
||||
}
|
||||
|
||||
ntb->reg = &xeon_gen3_reg;
|
||||
ntb->self_reg = &xeon_gen3_pri_reg;
|
||||
ntb->peer_reg = &xeon_gen3_b2b_reg;
|
||||
ntb->xlat_reg = &xeon_gen3_sec_xlat;
|
||||
|
||||
ntb->db_valid_mask = (1ULL << ntb->db_count) - 1;
|
||||
|
||||
xeon_gen3_setup_b2b_mw(ntb);
|
||||
|
||||
/* Enable Bus Master and Memory Space on the External Side */
|
||||
intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD,
|
||||
PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN);
|
||||
|
||||
/* Setup Interrupt */
|
||||
rc = intel_ntb_xeon_gen3_init_isr(ntb);
|
||||
|
||||
return (rc);
|
||||
}
|
||||
|
||||
static int
|
||||
intel_ntb_atom_init_dev(struct ntb_softc *ntb)
|
||||
{
|
||||
@ -1908,6 +2131,50 @@ xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr,
|
||||
return (0);
|
||||
}
|
||||
|
||||
static int
|
||||
xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb)
|
||||
{
|
||||
uint64_t reg;
|
||||
uint32_t embarsz, imbarsz;
|
||||
|
||||
/* IMBAR1SZ should be equal to EMBAR1SZ */
|
||||
embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1);
|
||||
imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1);
|
||||
if (embarsz != imbarsz) {
|
||||
device_printf(ntb->device,
|
||||
"IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n",
|
||||
imbarsz, embarsz);
|
||||
return (EIO);
|
||||
}
|
||||
|
||||
/* IMBAR2SZ should be equal to EMBAR2SZ */
|
||||
embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1);
|
||||
imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1);
|
||||
if (embarsz != imbarsz) {
|
||||
device_printf(ntb->device,
|
||||
"IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n",
|
||||
imbarsz, embarsz);
|
||||
return (EIO);
|
||||
}
|
||||
|
||||
/* Client will provide the incoming IMBAR1/2XBASE, zero it for now */
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0);
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0);
|
||||
|
||||
/*
|
||||
* If the value in EMBAR1LIMIT is set equal to the value in EMBAR1,
|
||||
* the memory window for EMBAR1 is disabled.
|
||||
* Note: It is needed to avoid malacious access.
|
||||
*/
|
||||
reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR1BASE, 8);
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, reg);
|
||||
|
||||
reg = pci_read_config(ntb->device, XEON_GEN3_EXT_REG_BAR2BASE, 8);
|
||||
intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, reg);
|
||||
|
||||
return (0);
|
||||
}
|
||||
|
||||
static inline bool
|
||||
_xeon_link_is_up(struct ntb_softc *ntb)
|
||||
{
|
||||
@ -1921,7 +2188,7 @@ static inline bool
|
||||
link_is_up(struct ntb_softc *ntb)
|
||||
{
|
||||
|
||||
if (ntb->type == NTB_XEON)
|
||||
if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3)
|
||||
return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good ||
|
||||
!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)));
|
||||
|
||||
@ -2185,7 +2452,9 @@ intel_ntb_poll_link(struct ntb_softc *ntb)
|
||||
ntb->ntb_ctl = ntb_cntl;
|
||||
ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta);
|
||||
} else {
|
||||
db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask);
|
||||
if (ntb->type == NTB_XEON_GEN1)
|
||||
db_iowrite_raw(ntb, ntb->self_reg->db_bell,
|
||||
ntb->db_link_mask);
|
||||
|
||||
reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2);
|
||||
if (reg_val == ntb->lnk_sta)
|
||||
@ -2960,6 +3229,8 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
|
||||
|
||||
if (limit_reg != 0 && size != mw_size)
|
||||
limit = base + size;
|
||||
else
|
||||
limit = base + mw_size;
|
||||
|
||||
/* Set and verify translation address */
|
||||
intel_ntb_reg_write(8, xlat_reg, addr);
|
||||
@ -2977,8 +3248,22 @@ intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size)
|
||||
intel_ntb_reg_write(8, xlat_reg, 0);
|
||||
return (EIO);
|
||||
}
|
||||
|
||||
if (ntb->type == NTB_XEON_GEN3) {
|
||||
limit = base + size;
|
||||
|
||||
/* set EMBAR1/2XLIMIT */
|
||||
if (!idx)
|
||||
intel_ntb_reg_write(8,
|
||||
XEON_GEN3_REG_EMBAR1XLIMIT, limit);
|
||||
else
|
||||
intel_ntb_reg_write(8,
|
||||
XEON_GEN3_REG_EMBAR2XLIMIT, limit);
|
||||
}
|
||||
} else {
|
||||
/* Configure 32-bit (split) BAR MW */
|
||||
if (ntb->type == NTB_XEON_GEN3)
|
||||
return (EIO);
|
||||
|
||||
if ((addr & UINT32_MAX) != addr)
|
||||
return (ERANGE);
|
||||
@ -3062,9 +3347,15 @@ intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t m
|
||||
}
|
||||
|
||||
static void
|
||||
intel_ntb_peer_db_set(device_t dev, uint64_t bit)
|
||||
intel_ntb_peer_db_set(device_t dev, uint64_t bits)
|
||||
{
|
||||
struct ntb_softc *ntb = device_get_softc(dev);
|
||||
uint64_t db;
|
||||
|
||||
if ((bits & ~ntb->db_valid_mask) != 0) {
|
||||
device_printf(ntb->device, "Invalid doorbell bits %lx\n", bits);
|
||||
return;
|
||||
}
|
||||
|
||||
if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) {
|
||||
struct ntb_pci_bar_info *lapic;
|
||||
@ -3073,7 +3364,7 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
|
||||
lapic = ntb->peer_lapic_bar;
|
||||
|
||||
for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) {
|
||||
if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0)
|
||||
if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0)
|
||||
bus_space_write_4(lapic->pci_bus_tag,
|
||||
lapic->pci_bus_handle,
|
||||
ntb->peer_msix_data[i].nmd_ofs,
|
||||
@ -3083,11 +3374,22 @@ intel_ntb_peer_db_set(device_t dev, uint64_t bit)
|
||||
}
|
||||
|
||||
if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) {
|
||||
intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit);
|
||||
intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits);
|
||||
return;
|
||||
}
|
||||
|
||||
db_iowrite(ntb, ntb->peer_reg->db_bell, bit);
|
||||
if (ntb->type == NTB_XEON_GEN3) {
|
||||
while (bits != 0) {
|
||||
db = ffsll(bits);
|
||||
|
||||
intel_ntb_reg_write(1,
|
||||
ntb->peer_reg->db_bell + (db - 1) * 4, 0x1);
|
||||
|
||||
bits = bits & (bits - 1);
|
||||
}
|
||||
} else {
|
||||
db_iowrite(ntb, ntb->peer_reg->db_bell, bits);
|
||||
}
|
||||
}
|
||||
|
||||
static int
|
||||
|
||||
@ -30,6 +30,25 @@
|
||||
|
||||
#ifndef _NTB_REGS_H_
|
||||
#define _NTB_REGS_H_
|
||||
#include <sys/types.h>
|
||||
#include <sys/stdint.h>
|
||||
|
||||
/*---------------------------------------------------------------------------
|
||||
* Macro: M*_M : Create a mask to isolate a bit field of a data word.
|
||||
* M*_F : Extract value from a bit field of a data word.
|
||||
* M*_I : Insert value into a bit field of a data word.
|
||||
*
|
||||
* Purpose: Bit field manipulation macros for mask, insert and extract for
|
||||
* 8-bit, 16-bit, 32-bit and 64-bit data words.
|
||||
*
|
||||
* Params: [in] P = Bit position of start of the bit field (lsb is 0).
|
||||
* [in] N = Size of the bit field in bits.
|
||||
* [in] X = Value to insert or remove from the bit field.
|
||||
*---------------------------------------------------------------------------
|
||||
*/
|
||||
#define M8_M(P, N) ((UINT8_MAX >> (8 - (N))) << (P))
|
||||
#define M8_F(X, P, N) (((uint8_t)(X) & M8_M(P, N)) >> (P))
|
||||
#define M8_I(X, P, N) (((uint8_t)(X) << (P)) & M8_M(P, N))
|
||||
|
||||
#define NTB_LINK_STATUS_ACTIVE 0x2000
|
||||
#define NTB_LINK_SPEED_MASK 0x000f
|
||||
@ -164,5 +183,73 @@
|
||||
|
||||
/* The peer ntb secondary config space is 32KB fixed size */
|
||||
#define XEON_B2B_MIN_SIZE 0x8000
|
||||
#define XEON_GEN3_MW_COUNT 2
|
||||
#define XEON_GEN3_SPLIT_MW_COUNT 3
|
||||
#define XEON_GEN3_SPAD_COUNT 16
|
||||
#define XEON_GEN3_DB_COUNT 32
|
||||
#define XEON_GEN3_DB_LINK 32
|
||||
#define XEON_GEN3_DB_LINK_BIT (1ULL << XEON_GEN3_DB_LINK)
|
||||
#define XEON_GEN3_DB_MSIX_VECTOR_COUNT 33
|
||||
#define XEON_GEN3_DB_MSIX_VECTOR_SHIFT 1
|
||||
|
||||
#define XEON_GEN3_LINK_VECTOR_INDEX 31
|
||||
|
||||
/* Xeon Skylake NTB register definitions */
|
||||
|
||||
/*
|
||||
* Internal EndPoint Configuration Registers
|
||||
*/
|
||||
#define XEON_GEN3_INT_REG_BAR0BASE 0x10
|
||||
#define XEON_GEN3_INT_REG_BAR1BASE 0x18
|
||||
#define XEON_GEN3_INT_REG_BAR2BASE 0x20
|
||||
#define XEON_GEN3_INT_REG_IMBAR1SZ 0xd0
|
||||
#define XEON_GEN3_INT_REG_IMBAR2SZ 0xd1
|
||||
#define XEON_GEN3_INT_REG_EMBAR1SZ 0xd2
|
||||
#define XEON_GEN3_INT_REG_EMBAR2SZ 0xd3
|
||||
#define XEON_GEN3_INT_REG_PPD 0xd4
|
||||
#define XEON_GEN3_INT_LNK_STS_OFFSET 0x01a2
|
||||
|
||||
/*
|
||||
* External EndPoint Configuration Registers
|
||||
* These are located within BAR0 of the internal endpoint.
|
||||
*/
|
||||
#define XEON_GEN3_EXT_REG_PCI_CMD 0x4504
|
||||
#define XEON_GEN3_EXT_REG_BAR0BASE 0x4510
|
||||
#define XEON_GEN3_EXT_REG_BAR1BASE 0x4518
|
||||
#define XEON_GEN3_EXT_REG_BAR2BASE 0x4520
|
||||
|
||||
/*
|
||||
* Internal Endpoint Memory Mapped Registers
|
||||
*/
|
||||
#define XEON_GEN3_REG_IMNTB_CTRL 0x0000
|
||||
#define XEON_GEN3_REG_IMBAR1XBASE 0x0010
|
||||
#define XEON_GEN3_REG_IMBAR1XLIMIT 0x0018
|
||||
#define XEON_GEN3_REG_IMBAR2XBASE 0x0020
|
||||
#define XEON_GEN3_REG_IMBAR2XLIMIT 0x0028
|
||||
#define XEON_GEN3_REG_IMINT_STATUS 0x0040
|
||||
#define XEON_GEN3_REG_IMINT_DISABLE 0x0048
|
||||
#define XEON_GEN3_REG_IMSPAD 0x0080
|
||||
#define XEON_GEN3_REG_IMINTVEC00 0x00d0
|
||||
#define XEON_GEN3_REG_IMDOORBELL 0x0100
|
||||
#define XEON_GEN3_REG_IMB2B_SSPAD 0x0180 /* Pseudo SP registers */
|
||||
|
||||
/*
|
||||
* External Endpoint Memory Mapped Registers
|
||||
*/
|
||||
#define XEON_GEN3_REG_EMBAR0XBASE 0x4008
|
||||
#define XEON_GEN3_REG_EMBAR1XBASE 0x4010
|
||||
#define XEON_GEN3_REG_EMBAR1XLIMIT 0x4018
|
||||
#define XEON_GEN3_REG_EMBAR2XBASE 0x4020
|
||||
#define XEON_GEN3_REG_EMBAR2XLIMIT 0x4028
|
||||
#define XEON_GEN3_REG_EMINT_STATUS 0x4040
|
||||
#define XEON_GEN3_REG_EMINT_DISABLE 0x4048
|
||||
#define XEON_GEN3_REG_EMSPAD 0x4080
|
||||
#define XEON_GEN3_REG_EMDOORBELL 0x4100
|
||||
|
||||
/* XEON_GEN3_INT_REG_PPD: PPD register */
|
||||
#define XEON_GEN3_REG_PPD_PORT_DEF_F(X) M8_F(X, 0, 2)
|
||||
#define XEON_GEN3_REG_PPD_CONF_STS_F(X) M8_F(X, 4, 1)
|
||||
#define XEON_GEN3_REG_PPD_ONE_MSIX_F(X) M8_F(X, 5, 1)
|
||||
#define XEON_GEN3_REG_PPD_BAR45_SPL_F(X) M8_F(X, 6, 1)
|
||||
|
||||
#endif /* _NTB_REGS_H_ */
|
||||
|
||||
Loading…
Reference in New Issue
Block a user